src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31             Edwin Steiner
  32
  33    $Id: codegen.c 4791 2006-04-18 21:16:36Z twisti $
  34
  35 */
  36
  37
  38 #include "config.h"
  39
  40 #include <assert.h>
  41 #include <stdio.h>
  42
  43 #include "vm/types.h"
  44
  45 #include "md.h"
  46 #include "md-abi.h"
  47
  48 #include "vm/jit/x86_64/arch.h"
  49 #include "vm/jit/x86_64/codegen.h"
  50 #include "vm/jit/x86_64/md-emit.h"
  51
  52 #include "mm/memory.h"
  53 #include "native/jni.h"
  54 #include "native/native.h"
  55 #include "vm/builtin.h"
  56 #include "vm/exceptions.h"
  57 #include "vm/global.h"
  58 #include "vm/loader.h"
  59 #include "vm/options.h"
  60 #include "vm/statistics.h"
  61 #include "vm/stringlocal.h"
  62 #include "vm/vm.h"
  63 #include "vm/jit/asmpart.h"
  64 #include "vm/jit/codegen-common.h"
  65 #include "vm/jit/dseg.h"
  66 #include "vm/jit/emit.h"
  67 #include "vm/jit/jit.h"
  68 #include "vm/jit/methodheader.h"
  69 #include "vm/jit/parse.h"
  70 #include "vm/jit/patcher.h"
  71 #include "vm/jit/reg.h"
  72 #include "vm/jit/replace.h"
  73
  74 #if defined(ENABLE_LSRA)
  75 # include "vm/jit/allocator/lsra.h"
  76 #endif
  77
  78
  79 /* codegen *********************************************************************
  80
  81    Generates machine code.
  82
  83 *******************************************************************************/
  84
  85 bool codegen(jitdata *jd)
  86 {
  87         methodinfo         *m;
  88         codegendata        *cd;
  89         registerdata       *rd;
  90         s4                  len, s1, s2, s3, d, disp;
  91         u2                  currentline;
  92         ptrint              a;
  93         s4                  stackframesize;
  94         stackptr            src;
  95         varinfo            *var;
  96         basicblock         *bptr;
  97         instruction        *iptr;
  98         exceptiontable     *ex;
  99         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 100         builtintable_entry *bte;
 101         methoddesc         *md;
 102         rplpoint           *replacementpoint;
 103
 104         /* get required compiler data */
 105
 106         m  = jd->m;
 107         cd = jd->cd;
 108         rd = jd->rd;
 109
 110         /* prevent compiler warnings */
 111
 112         d = 0;
 113         lm = NULL;
 114         bte = NULL;
 115
 116         {
 117         s4 i, p, t, l;
 118         s4 savedregs_num;
 119
 120         savedregs_num = 0;
 121
 122         /* space to save used callee saved registers */
 123
 124         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 125         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 126
 127         stackframesize = rd->memuse + savedregs_num;
 128
 129 #if defined(USE_THREADS)
 130         /* space to save argument of monitor_enter */
 131
 132         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 133                 stackframesize++;
 134 #endif
 135
 136     /* Keep stack of non-leaf functions 16-byte aligned for calls into native */
 137         /* code e.g. libc or jni (alignment problems with movaps).                */
 138
 139         if (!m->isleafmethod || opt_verbosecall)
 140                 stackframesize |= 0x1;
 141
 142         /* create method header */
 143
 144         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 145         (void) dseg_adds4(cd, stackframesize * 8);              /* FrameSize      */
 146
 147 #if defined(USE_THREADS)
 148         /* IsSync contains the offset relative to the stack pointer for the
 149            argument of monitor_exit used in the exception handler. Since the
 150            offset could be zero and give a wrong meaning of the flag it is
 151            offset by one.
 152         */
 153
 154         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 155                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);        /* IsSync         */
 156         else
 157 #endif
 158                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 159
 160         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 161         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse);  /* IntSave        */
 162         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse);  /* FltSave        */
 163
 164         (void) dseg_addlinenumbertablesize(cd);
 165
 166         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 167
 168         /* create exception table */
 169
 170         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 171                 dseg_addtarget(cd, ex->start);
 172                 dseg_addtarget(cd, ex->end);
 173                 dseg_addtarget(cd, ex->handler);
 174                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 175         }
 176
 177         /* generate method profiling code */
 178
 179         if (opt_prof) {
 180                 /* count frequency */
 181
 182                 M_MOV_IMM(m, REG_ITMP3);
 183                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(methodinfo, frequency));
 184
 185                 PROFILE_CYCLE_START;
 186         }
 187
 188         /* create stack frame (if necessary) */
 189
 190         if (stackframesize)
 191                 M_ASUB_IMM(stackframesize * 8, REG_SP);
 192
 193         /* save used callee saved registers */
 194
 195         p = stackframesize;
 196         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 197                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 198         }
 199         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 200                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 201         }
 202
 203         /* take arguments out of register or stack frame */
 204
 205         md = m->parseddesc;
 206
 207         for (p = 0, l = 0; p < md->paramcount; p++) {
 208                 t = md->paramtypes[p].type;
 209                 var = &(rd->locals[l][t]);
 210                 l++;
 211                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 212                         l++;
 213                 if (var->type < 0)
 214                         continue;
 215                 s1 = md->params[p].regoff;
 216                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 217                         s2 = rd->argintregs[s1];
 218                         if (!md->params[p].inmemory) {           /* register arguments    */
 219                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 220                                         M_INTMOVE(s2, var->regoff);
 221
 222                                 } else {                             /* reg arg -> spilled    */
 223                                     M_LST(s2, REG_SP, var->regoff * 8);
 224                                 }
 225
 226                         } else {                                 /* stack arguments       */
 227                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 228                                         /* + 8 for return address */
 229                                         M_LLD(var->regoff, REG_SP, (stackframesize + s1) * 8 + 8);
 230
 231                                 } else {                             /* stack arg -> spilled  */
 232                                         var->regoff = stackframesize + s1 + 1;
 233                                 }
 234                         }
 235
 236                 } else {                                     /* floating args         */
 237                         if (!md->params[p].inmemory) {           /* register arguments    */
 238                                 s2 = rd->argfltregs[s1];
 239                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 240                                         M_FLTMOVE(s2, var->regoff);
 241
 242                                 } else {                                         /* reg arg -> spilled    */
 243                                         M_DST(s2, REG_SP, var->regoff * 8);
 244                                 }
 245
 246                         } else {                                 /* stack arguments       */
 247                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 248                                         M_DLD(var->regoff, REG_SP, (stackframesize + s1) * 8 + 8);
 249
 250                                 } else {
 251                                         var->regoff = stackframesize + s1 + 1;
 252                                 }
 253                         }
 254                 }
 255         }  /* end for */
 256
 257         /* save monitorenter argument */
 258
 259 #if defined(USE_THREADS)
 260         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 261                 /* stack offset for monitor argument */
 262
 263                 s1 = rd->memuse;
 264
 265                 if (opt_verbosecall) {
 266                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 267
 268                         for (p = 0; p < INT_ARG_CNT; p++)
 269                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 270
 271                         for (p = 0; p < FLT_ARG_CNT; p++)
 272                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 273
 274                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 275                 }
 276
 277                 /* decide which monitor enter function to call */
 278
 279                 if (m->flags & ACC_STATIC) {
 280                         M_MOV_IMM(m->class, REG_ITMP1);
 281                         M_AST(REG_ITMP1, REG_SP, s1 * 8);
 282                         M_INTMOVE(REG_ITMP1, rd->argintregs[0]);
 283                         M_MOV_IMM(BUILTIN_staticmonitorenter, REG_ITMP1);
 284                         M_CALL(REG_ITMP1);
 285
 286                 } else {
 287                         M_TEST(rd->argintregs[0]);
 288                         M_BEQ(0);
 289                         codegen_add_nullpointerexception_ref(cd, cd->mcodeptr);
 290                         M_AST(rd->argintregs[0], REG_SP, s1 * 8);
 291                         M_MOV_IMM(BUILTIN_monitorenter, REG_ITMP1);
 292                         M_CALL(REG_ITMP1);
 293                 }
 294
 295                 if (opt_verbosecall) {
 296                         for (p = 0; p < INT_ARG_CNT; p++)
 297                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 298
 299                         for (p = 0; p < FLT_ARG_CNT; p++)
 300                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 301
 302                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 303                 }
 304         }
 305 #endif
 306
 307 #if !defined(NDEBUG)
 308         /* Copy argument registers to stack and call trace function with
 309            pointer to arguments on stack. */
 310
 311         if (opt_verbosecall) {
 312                 M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 313
 314                 /* save integer argument registers */
 315
 316                 for (p = 0; p < INT_ARG_CNT; p++)
 317                         M_LST(rd->argintregs[p], REG_SP, (1 + p) * 8);
 318
 319                 /* save float argument registers */
 320
 321                 for (p = 0; p < FLT_ARG_CNT; p++)
 322                         M_DST(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 323
 324                 /* save temporary registers for leaf methods */
 325
 326                 if (m->isleafmethod) {
 327                         for (p = 0; p < INT_TMP_CNT; p++)
 328                                 M_LST(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 329
 330                         for (p = 0; p < FLT_TMP_CNT; p++)
 331                                 M_DST(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 332                 }
 333
 334                 /* show integer hex code for float arguments */
 335
 336                 for (p = 0, l = 0; p < md->paramcount && p < INT_ARG_CNT; p++) {
 337                         /* If the paramtype is a float, we have to right shift all
 338                            following integer registers. */
 339
 340                         if (IS_FLT_DBL_TYPE(md->paramtypes[p].type)) {
 341                                 for (s1 = INT_ARG_CNT - 2; s1 >= p; s1--) {
 342                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
 343                                 }
 344
 345                                 x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 346                                 l++;
 347                         }
 348                 }
 349
 350                 M_MOV_IMM(m, REG_ITMP2);
 351                 M_AST(REG_ITMP2, REG_SP, 0 * 8);
 352                 M_MOV_IMM(builtin_trace_args, REG_ITMP1);
 353                 M_CALL(REG_ITMP1);
 354
 355                 /* restore integer argument registers */
 356
 357                 for (p = 0; p < INT_ARG_CNT; p++)
 358                         M_LLD(rd->argintregs[p], REG_SP, (1 + p) * 8);
 359
 360                 /* restore float argument registers */
 361
 362                 for (p = 0; p < FLT_ARG_CNT; p++)
 363                         M_DLD(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 364
 365                 /* restore temporary registers for leaf methods */
 366
 367                 if (m->isleafmethod) {
 368                         for (p = 0; p < INT_TMP_CNT; p++)
 369                                 M_LLD(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 370
 371                         for (p = 0; p < FLT_TMP_CNT; p++)
 372                                 M_DLD(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 373                 }
 374
 375                 M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 376         }
 377 #endif /* !defined(NDEBUG) */
 378
 379         }
 380
 381         /* end of header generation */
 382
 383         replacementpoint = jd->code->rplpoints;
 384
 385         /* walk through all basic blocks */
 386
 387         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 388
 389                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 390
 391                 if (bptr->flags >= BBREACHED) {
 392
 393                         /* branch resolving */
 394
 395                         branchref *bref;
 396                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 397                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 398                                                                   bref->branchpos,
 399                                                                   bptr->mpc);
 400                         }
 401
 402                 /* handle replacement points */
 403
 404                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 405                         replacementpoint->pc = (u1*)(ptrint)bptr->mpc; /* will be resolved later */
 406
 407                         replacementpoint++;
 408
 409                         assert(cd->lastmcodeptr <= cd->mcodeptr);
 410                         cd->lastmcodeptr = cd->mcodeptr + 5; /* 5 byte jmp patch */
 411                 }
 412
 413                 /* copy interface registers to their destination */
 414
 415                 src = bptr->instack;
 416                 len = bptr->indepth;
 417                 MCODECHECK(512);
 418
 419                 /* generate basicblock profiling code */
 420
 421                 if (opt_prof_bb) {
 422                         /* count frequency */
 423
 424                         M_MOV_IMM(m->bbfrequency, REG_ITMP2);
 425                         M_IINC_MEMBASE(REG_ITMP2, bptr->debug_nr * 4);
 426
 427                         /* if this is an exception handler, start profiling again */
 428
 429                         if (bptr->type == BBTYPE_EXH)
 430                                 PROFILE_CYCLE_START;
 431                 }
 432
 433 #if defined(ENABLE_LSRA)
 434                 if (opt_lsra) {
 435                         while (src != NULL) {
 436                                 len--;
 437                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 438                                         if (bptr->type == BBTYPE_SBR) {
 439 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 440                                                 if (!(src->flags & INMEMORY))
 441                                                         d= src->regoff;
 442                                                 else
 443                                                         d=REG_ITMP1;
 444                                                 x86_64_pop_reg(cd, d);
 445                                                 emit_store(jd, NULL, src, d);
 446
 447                                         } else if (bptr->type == BBTYPE_EXH) {
 448 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 449                                                 if (!(src->flags & INMEMORY))
 450                                                         d= src->regoff;
 451                                                 else
 452                                                         d=REG_ITMP1;
 453                                                 M_INTMOVE(REG_ITMP1, d);
 454                                                 emit_store(jd, NULL, src, d);
 455                                         }
 456                                 }
 457                                 src = src->prev;
 458                         }
 459
 460                 } else {
 461 #endif
 462
 463                 while (src != NULL) {
 464                         len--;
 465                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 466                                 if (bptr->type == BBTYPE_SBR) {
 467                                         d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 468                                         M_POP(d);
 469                                         emit_store(jd, NULL, src, d);
 470
 471                                 } else if (bptr->type == BBTYPE_EXH) {
 472                                         d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 473                                         M_INTMOVE(REG_ITMP1, d);
 474                                         emit_store(jd, NULL, src, d);
 475                                 }
 476
 477                         } else {
 478                                 d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 479                                 if ((src->varkind != STACKVAR)) {
 480                                         s2 = src->type;
 481                                         if (IS_FLT_DBL_TYPE(s2)) {
 482                                                 s1 = rd->interfaces[len][s2].regoff;
 483
 484                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 485                                                         M_FLTMOVE(s1, d);
 486                                                 else
 487                                                         M_DLD(d, REG_SP, s1 * 8);
 488
 489                                                 emit_store(jd, NULL, src, d);
 490
 491                                         } else {
 492                                                 s1 = rd->interfaces[len][s2].regoff;
 493
 494                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 495                                                         M_INTMOVE(s1, d);
 496                                                 else
 497                                                         M_LLD(d, REG_SP, s1 * 8);
 498
 499                                                 emit_store(jd, NULL, src, d);
 500                                         }
 501                                 }
 502                         }
 503                         src = src->prev;
 504                 }
 505 #if defined(ENABLE_LSRA)
 506                 }
 507 #endif
 508                 /* walk through all instructions */
 509
 510                 src = bptr->instack;
 511                 len = bptr->icount;
 512                 currentline = 0;
 513
 514                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 515                         if (iptr->line != currentline) {
 516                                 dseg_addlinenumber(cd, iptr->line, cd->mcodeptr);
 517                                 currentline = iptr->line;
 518                         }
 519
 520                         MCODECHECK(1024);                         /* 1KB should be enough */
 521
 522                         switch (iptr->opc) {
 523                         case ICMD_INLINE_START: /* internal ICMDs                         */
 524                         case ICMD_INLINE_END:
 525                                 break;
 526
 527                         case ICMD_NOP:    /* ...  ==> ...                                 */
 528                                 break;
 529
 530                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 531
 532                                 if (src->flags & INMEMORY)
 533                                         M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
 534                                 else
 535                                         M_TEST(src->regoff);
 536                                 M_BEQ(0);
 537                                 codegen_add_nullpointerexception_ref(cd, cd->mcodeptr);
 538                                 break;
 539
 540                 /* constant operations ************************************************/
 541
 542                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 543                                       /* op1 = 0, val.i = constant                    */
 544
 545                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 546                         ICONST(d, iptr->val.i);
 547                         emit_store(jd, iptr, iptr->dst, d);
 548                         break;
 549
 550                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 551                                       /* op1 = 0, val.l = constant                    */
 552
 553                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 554                         LCONST(d, iptr->val.l);
 555                         emit_store(jd, iptr, iptr->dst, d);
 556                         break;
 557
 558                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 559                                       /* op1 = 0, val.f = constant                    */
 560
 561                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 562                         disp = dseg_addfloat(cd, iptr->val.f);
 563                         x86_64_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + disp, d);
 564                         emit_store(jd, iptr, iptr->dst, d);
 565                         break;
 566
 567                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 568                                       /* op1 = 0, val.d = constant                    */
 569
 570                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 571                         disp = dseg_adddouble(cd, iptr->val.d);
 572                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, d);
 573                         emit_store(jd, iptr, iptr->dst, d);
 574                         break;
 575
 576                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 577                                       /* op1 = 0, val.a = constant                    */
 578
 579                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 580
 581                         if ((iptr->target != NULL) && (iptr->val.a == NULL)) {
 582 /*                              PROFILE_CYCLE_STOP; */
 583
 584                                 codegen_addpatchref(cd, cd->mcodeptr,
 585                                                                         PATCHER_aconst,
 586                                                                         (unresolved_class *) iptr->target, 0);
 587
 588                                 if (opt_showdisassemble) {
 589                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
 590                                 }
 591
 592 /*                              PROFILE_CYCLE_START; */
 593
 594                                 M_MOV_IMM(iptr->val.a, d);
 595
 596                         } else {
 597                                 if (iptr->val.a == 0)
 598                                         M_CLR(d);
 599                                 else
 600                                         M_MOV_IMM(iptr->val.a, d);
 601                         }
 602                         emit_store(jd, iptr, iptr->dst, d);
 603                         break;
 604
 605
 606                 /* load/store operations **********************************************/
 607
 608                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 609                                       /* op1 = local variable                         */
 610
 611                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 612                         if ((iptr->dst->varkind == LOCALVAR) &&
 613                                 (iptr->dst->varnum == iptr->op1))
 614                                 break;
 615                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 616                         if (var->flags & INMEMORY)
 617                                 M_ILD(d, REG_SP, var->regoff * 8);
 618                         else
 619                                 M_INTMOVE(var->regoff, d);
 620                         emit_store(jd, iptr, iptr->dst, d);
 621                         break;
 622
 623                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 624                 case ICMD_ALOAD:      /* op1 = local variable                         */
 625
 626                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 627                         if ((iptr->dst->varkind == LOCALVAR) &&
 628                                 (iptr->dst->varnum == iptr->op1))
 629                                 break;
 630                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 631                         if (var->flags & INMEMORY)
 632                                 M_LLD(d, REG_SP, var->regoff * 8);
 633                         else
 634                                 M_INTMOVE(var->regoff, d);
 635                         emit_store(jd, iptr, iptr->dst, d);
 636                         break;
 637
 638                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 639                 case ICMD_DLOAD:      /* op1 = local variable                         */
 640
 641                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 642                         if ((iptr->dst->varkind == LOCALVAR) &&
 643                             (iptr->dst->varnum == iptr->op1))
 644                                 break;
 645                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 646                         if (var->flags & INMEMORY)
 647                                 M_DLD(d, REG_SP, var->regoff * 8);
 648                         else
 649                                 M_FLTMOVE(var->regoff, d);
 650                         emit_store(jd, iptr, iptr->dst, d);
 651                         break;
 652
 653                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 654                                       /* op1 = local variable                         */
 655
 656                         if ((src->varkind == LOCALVAR) && (src->varnum == iptr->op1))
 657                                 break;
 658                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 659                         if (var->flags & INMEMORY) {
 660                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 661                                 M_IST(s1, REG_SP, var->regoff * 8);
 662                         } else {
 663                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 664                                 M_INTMOVE(s1, var->regoff);
 665                         }
 666                         break;
 667
 668                 case ICMD_LSTORE:     /* ..., value  ==> ...                          */
 669                 case ICMD_ASTORE:     /* op1 = local variable                         */
 670
 671                         if ((src->varkind == LOCALVAR) && (src->varnum == iptr->op1))
 672                                 break;
 673                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 674                         if (var->flags & INMEMORY) {
 675                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 676                                 M_LST(s1, REG_SP, var->regoff * 8);
 677                         } else {
 678                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 679                                 M_INTMOVE(s1, var->regoff);
 680                         }
 681                         break;
 682
 683                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 684                 case ICMD_DSTORE:     /* op1 = local variable                         */
 685
 686                         if ((src->varkind == LOCALVAR) &&
 687                             (src->varnum == iptr->op1)) {
 688                                 break;
 689                         }
 690                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 691                         if (var->flags & INMEMORY) {
 692                                 s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
 693                                 M_DST(s1, REG_SP, var->regoff * 8);
 694                         } else {
 695                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 696                                 M_FLTMOVE(s1, var->regoff);
 697                         }
 698                         break;
 699
 700
 701                 /* pop/dup/swap operations ********************************************/
 702
 703                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 704
 705                 case ICMD_POP:        /* ..., value  ==> ...                          */
 706                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 707                         break;
 708
 709                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 710                         M_COPY(src, iptr->dst);
 711                         break;
 712
 713                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 714
 715                         M_COPY(src,       iptr->dst);
 716                         M_COPY(src->prev, iptr->dst->prev);
 717                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 718                         break;
 719
 720                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 721
 722                         M_COPY(src,             iptr->dst);
 723                         M_COPY(src->prev,       iptr->dst->prev);
 724                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 725                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 726                         break;
 727
 728                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 729
 730                         M_COPY(src,       iptr->dst);
 731                         M_COPY(src->prev, iptr->dst->prev);
 732                         break;
 733
 734                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 735
 736                         M_COPY(src,             iptr->dst);
 737                         M_COPY(src->prev,       iptr->dst->prev);
 738                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 739                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 740                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 741                         break;
 742
 743                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 744
 745                         M_COPY(src,                   iptr->dst);
 746                         M_COPY(src->prev,             iptr->dst->prev);
 747                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 748                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 749                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 750                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 751                         break;
 752
 753                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 754
 755                         M_COPY(src,       iptr->dst->prev);
 756                         M_COPY(src->prev, iptr->dst);
 757                         break;
 758
 759
 760                 /* integer operations *************************************************/
 761
 762                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 763
 764                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 765                         if (iptr->dst->flags & INMEMORY) {
 766                                 if (src->flags & INMEMORY) {
 767                                         if (src->regoff == iptr->dst->regoff)
 768                                                 M_INEG_MEMBASE(REG_SP, iptr->dst->regoff * 8);
 769                                         else {
 770                                                 M_ILD(REG_ITMP1, REG_SP, src->regoff * 8);
 771                                                 M_INEG(REG_ITMP1);
 772                                                 M_IST(REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 773                                         }
 774
 775                                 } else {
 776                                         M_IST(src->regoff, REG_SP, iptr->dst->regoff * 8);
 777                                         M_INEG_MEMBASE(REG_SP, iptr->dst->regoff * 8);
 778                                 }
 779
 780                         } else {
 781                                 if (src->flags & INMEMORY) {
 782                                         M_ILD(iptr->dst->regoff, REG_SP, src->regoff * 8);
 783                                         M_INEG(iptr->dst->regoff);
 784
 785                                 } else {
 786                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 787                                         M_INEG(iptr->dst->regoff);
 788                                 }
 789                         }
 790                         break;
 791
 792                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 793
 794                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 795                         if (iptr->dst->flags & INMEMORY) {
 796                                 if (src->flags & INMEMORY) {
 797                                         if (src->regoff == iptr->dst->regoff)
 798                                                 M_LNEG_MEMBASE(REG_SP, iptr->dst->regoff * 8);
 799                                         else {
 800                                                 M_LLD(REG_ITMP1, REG_SP, src->regoff * 8);
 801                                                 M_LNEG(REG_ITMP1);
 802                                                 M_LST(REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 803                                         }
 804
 805                                 } else {
 806                                         M_LST(src->regoff, REG_SP, iptr->dst->regoff * 8);
 807                                         M_LNEG_MEMBASE(REG_SP, iptr->dst->regoff * 8);
 808                                 }
 809
 810                         } else {
 811                                 if (src->flags & INMEMORY) {
 812                                         M_LLD(iptr->dst->regoff, REG_SP, src->regoff * 8);
 813                                         M_LNEG(iptr->dst->regoff);
 814
 815                                 } else {
 816                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 817                                         M_LNEG(iptr->dst->regoff);
 818                                 }
 819                         }
 820                         break;
 821
 822                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 823
 824                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 825                         if (src->flags & INMEMORY) {
 826                                 x86_64_movslq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 827
 828                         } else {
 829                                 x86_64_movslq_reg_reg(cd, src->regoff, d);
 830                         }
 831                         emit_store(jd, iptr, iptr->dst, d);
 832                         break;
 833
 834                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 835
 836                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 837                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 838                         M_INTMOVE(s1, d);
 839                         emit_store(jd, iptr, iptr->dst, d);
 840                         break;
 841
 842                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 843
 844                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 845                         if (src->flags & INMEMORY) {
 846                                 x86_64_movsbq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 847
 848                         } else {
 849                                 x86_64_movsbq_reg_reg(cd, src->regoff, d);
 850                         }
 851                         emit_store(jd, iptr, iptr->dst, d);
 852                         break;
 853
 854                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 855
 856                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 857                         if (src->flags & INMEMORY) {
 858                                 x86_64_movzwq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 859
 860                         } else {
 861                                 x86_64_movzwq_reg_reg(cd, src->regoff, d);
 862                         }
 863                         emit_store(jd, iptr, iptr->dst, d);
 864                         break;
 865
 866                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 867
 868                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 869                         if (src->flags & INMEMORY) {
 870                                 x86_64_movswq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 871
 872                         } else {
 873                                 x86_64_movswq_reg_reg(cd, src->regoff, d);
 874                         }
 875                         emit_store(jd, iptr, iptr->dst, d);
 876                         break;
 877
 878
 879                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 880
 881                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 882                         x86_64_emit_ialu(cd, X86_64_ADD, src, iptr);
 883                         break;
 884
 885                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 886                                       /* val.i = constant                             */
 887
 888                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 889                         x86_64_emit_ialuconst(cd, X86_64_ADD, src, iptr);
 890                         break;
 891
 892                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 893
 894                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 895                         x86_64_emit_lalu(cd, X86_64_ADD, src, iptr);
 896                         break;
 897
 898                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 899                                       /* val.l = constant                             */
 900
 901                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 902                         x86_64_emit_laluconst(cd, X86_64_ADD, src, iptr);
 903                         break;
 904
 905                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 906
 907                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 908                         if (iptr->dst->flags & INMEMORY) {
 909                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 910                                         if (src->prev->regoff == iptr->dst->regoff) {
 911                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 912                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 913
 914                                         } else {
 915                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 916                                                 x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 917                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 918                                         }
 919
 920                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 921                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 922                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 923                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 924
 925                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 926                                         if (src->prev->regoff == iptr->dst->regoff) {
 927                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 928
 929                                         } else {
 930                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 931                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 932                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 933                                         }
 934
 935                                 } else {
 936                                         x86_64_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 937                                         x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 938                                 }
 939
 940                         } else {
 941                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 942                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 943                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 944
 945                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 946                                         M_INTMOVE(src->prev->regoff, d);
 947                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 948
 949                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 950                                         /* workaround for reg alloc */
 951                                         if (src->regoff == iptr->dst->regoff) {
 952                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 953                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 954                                                 M_INTMOVE(REG_ITMP1, d);
 955
 956                                         } else {
 957                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 958                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 959                                         }
 960
 961                                 } else {
 962                                         /* workaround for reg alloc */
 963                                         if (src->regoff == iptr->dst->regoff) {
 964                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
 965                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 966                                                 M_INTMOVE(REG_ITMP1, d);
 967
 968                                         } else {
 969                                                 M_INTMOVE(src->prev->regoff, d);
 970                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 971                                         }
 972                                 }
 973                         }
 974                         break;
 975
 976                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 977                                       /* val.i = constant                             */
 978
 979                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 980                         x86_64_emit_ialuconst(cd, X86_64_SUB, src, iptr);
 981                         break;
 982
 983                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 984
 985                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 986                         if (iptr->dst->flags & INMEMORY) {
 987                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 988                                         if (src->prev->regoff == iptr->dst->regoff) {
 989                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 990                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 991
 992                                         } else {
 993                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 994                                                 x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 995                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 996                                         }
 997
 998                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 999                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1000                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1001                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1002
1003                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1004                                         if (src->prev->regoff == iptr->dst->regoff) {
1005                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1006
1007                                         } else {
1008                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1009                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1010                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1011                                         }
1012
1013                                 } else {
1014                                         x86_64_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
1015                                         x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1016                                 }
1017
1018                         } else {
1019                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1020                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1021                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1022
1023                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1024                                         M_INTMOVE(src->prev->regoff, d);
1025                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1026
1027                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1028                                         /* workaround for reg alloc */
1029                                         if (src->regoff == iptr->dst->regoff) {
1030                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1031                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1032                                                 M_INTMOVE(REG_ITMP1, d);
1033
1034                                         } else {
1035                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1036                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1037                                         }
1038
1039                                 } else {
1040                                         /* workaround for reg alloc */
1041                                         if (src->regoff == iptr->dst->regoff) {
1042                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1043                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1044                                                 M_INTMOVE(REG_ITMP1, d);
1045
1046                                         } else {
1047                                                 M_INTMOVE(src->prev->regoff, d);
1048                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1049                                         }
1050                                 }
1051                         }
1052                         break;
1053
1054                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1055                                       /* val.l = constant                             */
1056
1057                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1058                         x86_64_emit_laluconst(cd, X86_64_SUB, src, iptr);
1059                         break;
1060
1061                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1062
1063                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1064                         if (iptr->dst->flags & INMEMORY) {
1065                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1066                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1067                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1068                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1069
1070                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1071                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1072                                         x86_64_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1073                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1074
1075                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1076                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1077                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1078                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1079
1080                                 } else {
1081                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1082                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1083                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1084                                 }
1085
1086                         } else {
1087                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1088                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1089                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1090
1091                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1092                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1093                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1094
1095                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1096                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1097                                         x86_64_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1098
1099                                 } else {
1100                                         if (src->regoff == iptr->dst->regoff) {
1101                                                 x86_64_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1102
1103                                         } else {
1104                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1105                                                 x86_64_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1106                                         }
1107                                 }
1108                         }
1109                         break;
1110
1111                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1112                                       /* val.i = constant                             */
1113
1114                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1115                         if (iptr->dst->flags & INMEMORY) {
1116                                 if (src->flags & INMEMORY) {
1117                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1118                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1119
1120                                 } else {
1121                                         x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1122                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1123                                 }
1124
1125                         } else {
1126                                 if (src->flags & INMEMORY) {
1127                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1128
1129                                 } else {
1130                                         if (iptr->val.i == 2) {
1131                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1132                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1133
1134                                         } else {
1135                                                 x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1136                                         }
1137                                 }
1138                         }
1139                         break;
1140
1141                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1142
1143                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1144                         if (iptr->dst->flags & INMEMORY) {
1145                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1146                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1147                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1148                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1149
1150                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1151                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1152                                         x86_64_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1153                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1154
1155                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1156                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1157                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1158                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1159
1160                                 } else {
1161                                         x86_64_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1162                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1163                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1164                                 }
1165
1166                         } else {
1167                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1168                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1169                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1170
1171                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1172                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1173                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1174
1175                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1176                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1177                                         x86_64_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1178
1179                                 } else {
1180                                         if (src->regoff == iptr->dst->regoff) {
1181                                                 x86_64_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1182
1183                                         } else {
1184                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1185                                                 x86_64_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1186                                         }
1187                                 }
1188                         }
1189                         break;
1190
1191                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1192                                       /* val.l = constant                             */
1193
1194                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1195                         if (iptr->dst->flags & INMEMORY) {
1196                                 if (src->flags & INMEMORY) {
1197                                         if (IS_IMM32(iptr->val.l)) {
1198                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1199
1200                                         } else {
1201                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1202                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1203                                         }
1204                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1205
1206                                 } else {
1207                                         if (IS_IMM32(iptr->val.l)) {
1208                                                 x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1209
1210                                         } else {
1211                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1212                                                 x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1213                                         }
1214                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1215                                 }
1216
1217                         } else {
1218                                 if (src->flags & INMEMORY) {
1219                                         if (IS_IMM32(iptr->val.l)) {
1220                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1221
1222                                         } else {
1223                                                 x86_64_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1224                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1225                                         }
1226
1227                                 } else {
1228                                         /* should match in many cases */
1229                                         if (iptr->val.l == 2) {
1230                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1231                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1232
1233                                         } else {
1234                                                 if (IS_IMM32(iptr->val.l)) {
1235                                                         x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1236
1237                                                 } else {
1238                                                         x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1239                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1240                                                         x86_64_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1241                                                 }
1242                                         }
1243                                 }
1244                         }
1245                         break;
1246
1247                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1248
1249                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1250                 if (src->prev->flags & INMEMORY) {
1251                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1252
1253                         } else {
1254                                 M_INTMOVE(src->prev->regoff, RAX);
1255                         }
1256
1257                         if (src->flags & INMEMORY) {
1258                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1259
1260                         } else {
1261                                 M_INTMOVE(src->regoff, REG_ITMP3);
1262                         }
1263                         gen_div_check(src);
1264
1265                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1266                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1267                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1268                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                /* 6 bytes */
1269
1270                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1271                         x86_64_cltd(cd);
1272                         x86_64_idivl_reg(cd, REG_ITMP3);
1273
1274                         if (iptr->dst->flags & INMEMORY) {
1275                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1276                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1277
1278                         } else {
1279                                 M_INTMOVE(RAX, iptr->dst->regoff);
1280
1281                                 if (iptr->dst->regoff != RDX) {
1282                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1283                                 }
1284                         }
1285                         break;
1286
1287                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1288                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1289                         if (src->prev->flags & INMEMORY) {
1290                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1291
1292                         } else {
1293                                 M_INTMOVE(src->prev->regoff, RAX);
1294                         }
1295
1296                         if (src->flags & INMEMORY) {
1297                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1298
1299                         } else {
1300                                 M_INTMOVE(src->regoff, REG_ITMP3);
1301                         }
1302                         gen_div_check(src);
1303
1304                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1305
1306                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1307                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1308
1309
1310                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1311                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1312                         x86_64_jcc(cd, X86_64_CC_E, 1 + 3);                    /* 6 bytes */
1313
1314                         x86_64_cltd(cd);
1315                         x86_64_idivl_reg(cd, REG_ITMP3);
1316
1317                         if (iptr->dst->flags & INMEMORY) {
1318                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1319                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1320
1321                         } else {
1322                                 M_INTMOVE(RDX, iptr->dst->regoff);
1323
1324                                 if (iptr->dst->regoff != RDX) {
1325                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1326                                 }
1327                         }
1328                         break;
1329
1330                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1331                                       /* val.i = constant                             */
1332
1333                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1334                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1335                         M_INTMOVE(s1, REG_ITMP1);
1336                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1337                         x86_64_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1338                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1339                         x86_64_shiftl_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1340                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1341                         emit_store(jd, iptr, iptr->dst, d);
1342                         break;
1343
1344                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1345                                       /* val.i = constant                             */
1346
1347                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1348                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1349                         M_INTMOVE(s1, REG_ITMP1);
1350                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1351                         x86_64_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1352                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1353                         x86_64_alul_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1354                         x86_64_alul_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1355                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1356                         emit_store(jd, iptr, iptr->dst, d);
1357                         break;
1358
1359
1360                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1361
1362                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1363
1364                 if (src->prev->flags & INMEMORY) {
1365                                 M_LLD(RAX, REG_SP, src->prev->regoff * 8);
1366
1367                         } else {
1368                                 M_INTMOVE(src->prev->regoff, RAX);
1369                         }
1370
1371                         if (src->flags & INMEMORY) {
1372                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1373
1374                         } else {
1375                                 M_INTMOVE(src->regoff, REG_ITMP3);
1376                         }
1377                         gen_div_check(src);
1378
1379                         /* check as described in jvm spec */
1380                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1381                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, RAX);
1382                         M_BNE(4 + 6);
1383                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1384                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
1385
1386                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1387                         x86_64_cqto(cd);
1388                         x86_64_idiv_reg(cd, REG_ITMP3);
1389
1390                         if (iptr->dst->flags & INMEMORY) {
1391                                 M_LST(RAX, REG_SP, iptr->dst->regoff * 8);
1392                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1393
1394                         } else {
1395                                 M_INTMOVE(RAX, iptr->dst->regoff);
1396
1397                                 if (iptr->dst->regoff != RDX) {
1398                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1399                                 }
1400                         }
1401                         break;
1402
1403                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1404
1405                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1406                         if (src->prev->flags & INMEMORY) {
1407                                 M_LLD(REG_ITMP1, REG_SP, src->prev->regoff * 8);
1408
1409                         } else {
1410                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1411                         }
1412
1413                         if (src->flags & INMEMORY) {
1414                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1415
1416                         } else {
1417                                 M_INTMOVE(src->regoff, REG_ITMP3);
1418                         }
1419                         gen_div_check(src);
1420
1421                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1422
1423                         /* check as described in jvm spec */
1424                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1425                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP1);
1426                         M_BNE(3 + 4 + 6);
1427
1428 #if 0
1429                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1430 #endif
1431                         M_XOR(RDX, RDX);                                       /* 3 bytes */
1432                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1433                         M_BEQ(2 + 3);                                          /* 6 bytes */
1434
1435                         x86_64_cqto(cd);
1436                         x86_64_idiv_reg(cd, REG_ITMP3);
1437
1438                         if (iptr->dst->flags & INMEMORY) {
1439                                 M_LST(RDX, REG_SP, iptr->dst->regoff * 8);
1440                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1441
1442                         } else {
1443                                 M_INTMOVE(RDX, iptr->dst->regoff);
1444
1445                                 if (iptr->dst->regoff != RDX) {
1446                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1447                                 }
1448                         }
1449                         break;
1450
1451                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1452                                       /* val.i = constant                             */
1453
1454                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1455                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1456                         M_INTMOVE(s1, REG_ITMP1);
1457                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1458                         x86_64_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1459                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1460                         x86_64_shift_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1461                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1462                         emit_store(jd, iptr, iptr->dst, d);
1463                         break;
1464
1465                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1466                                       /* val.l = constant                             */
1467
1468                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1469                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1470                         M_INTMOVE(s1, REG_ITMP1);
1471                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1472                         x86_64_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1473                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1474                         x86_64_alu_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1475                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1476                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1477                         emit_store(jd, iptr, iptr->dst, d);
1478                         break;
1479
1480                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1481
1482                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1483                         x86_64_emit_ishift(cd, X86_64_SHL, src, iptr);
1484                         break;
1485
1486                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1487                                       /* val.i = constant                             */
1488
1489                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1490                         x86_64_emit_ishiftconst(cd, X86_64_SHL, src, iptr);
1491                         break;
1492
1493                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1494
1495                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1496                         x86_64_emit_ishift(cd, X86_64_SAR, src, iptr);
1497                         break;
1498
1499                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1500                                       /* val.i = constant                             */
1501
1502                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1503                         x86_64_emit_ishiftconst(cd, X86_64_SAR, src, iptr);
1504                         break;
1505
1506                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1507
1508                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1509                         x86_64_emit_ishift(cd, X86_64_SHR, src, iptr);
1510                         break;
1511
1512                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1513                                       /* val.i = constant                             */
1514
1515                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1516                         x86_64_emit_ishiftconst(cd, X86_64_SHR, src, iptr);
1517                         break;
1518
1519                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1520
1521                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1522                         x86_64_emit_lshift(cd, X86_64_SHL, src, iptr);
1523                         break;
1524
1525         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1526                                           /* val.i = constant                             */
1527
1528                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1529                         x86_64_emit_lshiftconst(cd, X86_64_SHL, src, iptr);
1530                         break;
1531
1532                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1533
1534                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1535                         x86_64_emit_lshift(cd, X86_64_SAR, src, iptr);
1536                         break;
1537
1538                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1539                                       /* val.i = constant                             */
1540
1541                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1542                         x86_64_emit_lshiftconst(cd, X86_64_SAR, src, iptr);
1543                         break;
1544
1545                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1546
1547                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1548                         x86_64_emit_lshift(cd, X86_64_SHR, src, iptr);
1549                         break;
1550
1551                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1552                                       /* val.l = constant                             */
1553
1554                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1555                         x86_64_emit_lshiftconst(cd, X86_64_SHR, src, iptr);
1556                         break;
1557
1558                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1559
1560                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1561                         x86_64_emit_ialu(cd, X86_64_AND, src, iptr);
1562                         break;
1563
1564                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1565                                       /* val.i = constant                             */
1566
1567                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1568                         x86_64_emit_ialuconst(cd, X86_64_AND, src, iptr);
1569                         break;
1570
1571                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1572
1573                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1574                         x86_64_emit_lalu(cd, X86_64_AND, src, iptr);
1575                         break;
1576
1577                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1578                                       /* val.l = constant                             */
1579
1580                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1581                         x86_64_emit_laluconst(cd, X86_64_AND, src, iptr);
1582                         break;
1583
1584                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1585
1586                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1587                         x86_64_emit_ialu(cd, X86_64_OR, src, iptr);
1588                         break;
1589
1590                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1591                                       /* val.i = constant                             */
1592
1593                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1594                         x86_64_emit_ialuconst(cd, X86_64_OR, src, iptr);
1595                         break;
1596
1597                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1598
1599                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1600                         x86_64_emit_lalu(cd, X86_64_OR, src, iptr);
1601                         break;
1602
1603                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1604                                       /* val.l = constant                             */
1605
1606                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1607                         x86_64_emit_laluconst(cd, X86_64_OR, src, iptr);
1608                         break;
1609
1610                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1611
1612                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1613                         x86_64_emit_ialu(cd, X86_64_XOR, src, iptr);
1614                         break;
1615
1616                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1617                                       /* val.i = constant                             */
1618
1619                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1620                         x86_64_emit_ialuconst(cd, X86_64_XOR, src, iptr);
1621                         break;
1622
1623                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1624
1625                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1626                         x86_64_emit_lalu(cd, X86_64_XOR, src, iptr);
1627                         break;
1628
1629                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1630                                       /* val.l = constant                             */
1631
1632                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1633                         x86_64_emit_laluconst(cd, X86_64_XOR, src, iptr);
1634                         break;
1635
1636
1637                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1638                                       /* op1 = variable, val.i = constant             */
1639
1640                         /* using inc and dec is definitely faster than add -- tested      */
1641                         /* with sieve                                                     */
1642
1643                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1644                         d = var->regoff;
1645                         if (var->flags & INMEMORY) {
1646                                 if (iptr->val.i == 1) {
1647                                         x86_64_incl_membase(cd, REG_SP, d * 8);
1648
1649                                 } else if (iptr->val.i == -1) {
1650                                         x86_64_decl_membase(cd, REG_SP, d * 8);
1651
1652                                 } else {
1653                                         x86_64_alul_imm_membase(cd, X86_64_ADD, iptr->val.i, REG_SP, d * 8);
1654                                 }
1655
1656                         } else {
1657                                 if (iptr->val.i == 1) {
1658                                         x86_64_incl_reg(cd, d);
1659
1660                                 } else if (iptr->val.i == -1) {
1661                                         x86_64_decl_reg(cd, d);
1662
1663                                 } else {
1664                                         x86_64_alul_imm_reg(cd, X86_64_ADD, iptr->val.i, d);
1665                                 }
1666                         }
1667                         break;
1668
1669
1670                 /* floating operations ************************************************/
1671
1672                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1673
1674                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1675                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1676                         disp = dseg_adds4(cd, 0x80000000);
1677                         M_FLTMOVE(s1, d);
1678                         x86_64_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1679                         x86_64_xorps_reg_reg(cd, REG_FTMP2, d);
1680                         emit_store(jd, iptr, iptr->dst, d);
1681                         break;
1682
1683                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1684
1685                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1686                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1687                         disp = dseg_adds8(cd, 0x8000000000000000);
1688                         M_FLTMOVE(s1, d);
1689                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1690                         x86_64_xorpd_reg_reg(cd, REG_FTMP2, d);
1691                         emit_store(jd, iptr, iptr->dst, d);
1692                         break;
1693
1694                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1695
1696                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1697                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1698                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1699                         if (s1 == d) {
1700                                 x86_64_addss_reg_reg(cd, s2, d);
1701                         } else if (s2 == d) {
1702                                 x86_64_addss_reg_reg(cd, s1, d);
1703                         } else {
1704                                 M_FLTMOVE(s1, d);
1705                                 x86_64_addss_reg_reg(cd, s2, d);
1706                         }
1707                         emit_store(jd, iptr, iptr->dst, d);
1708                         break;
1709
1710                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1711
1712                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1713                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1714                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1715                         if (s1 == d) {
1716                                 x86_64_addsd_reg_reg(cd, s2, d);
1717                         } else if (s2 == d) {
1718                                 x86_64_addsd_reg_reg(cd, s1, d);
1719                         } else {
1720                                 M_FLTMOVE(s1, d);
1721                                 x86_64_addsd_reg_reg(cd, s2, d);
1722                         }
1723                         emit_store(jd, iptr, iptr->dst, d);
1724                         break;
1725
1726                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1727
1728                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1729                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1730                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1731                         if (s2 == d) {
1732                                 M_FLTMOVE(s2, REG_FTMP2);
1733                                 s2 = REG_FTMP2;
1734                         }
1735                         M_FLTMOVE(s1, d);
1736                         x86_64_subss_reg_reg(cd, s2, d);
1737                         emit_store(jd, iptr, iptr->dst, d);
1738                         break;
1739
1740                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1741
1742                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1743                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1744                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1745                         if (s2 == d) {
1746                                 M_FLTMOVE(s2, REG_FTMP2);
1747                                 s2 = REG_FTMP2;
1748                         }
1749                         M_FLTMOVE(s1, d);
1750                         x86_64_subsd_reg_reg(cd, s2, d);
1751                         emit_store(jd, iptr, iptr->dst, d);
1752                         break;
1753
1754                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1755
1756                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1757                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1758                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1759                         if (s1 == d) {
1760                                 x86_64_mulss_reg_reg(cd, s2, d);
1761                         } else if (s2 == d) {
1762                                 x86_64_mulss_reg_reg(cd, s1, d);
1763                         } else {
1764                                 M_FLTMOVE(s1, d);
1765                                 x86_64_mulss_reg_reg(cd, s2, d);
1766                         }
1767                         emit_store(jd, iptr, iptr->dst, d);
1768                         break;
1769
1770                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1771
1772                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1773                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1774                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1775                         if (s1 == d) {
1776                                 x86_64_mulsd_reg_reg(cd, s2, d);
1777                         } else if (s2 == d) {
1778                                 x86_64_mulsd_reg_reg(cd, s1, d);
1779                         } else {
1780                                 M_FLTMOVE(s1, d);
1781                                 x86_64_mulsd_reg_reg(cd, s2, d);
1782                         }
1783                         emit_store(jd, iptr, iptr->dst, d);
1784                         break;
1785
1786                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1787
1788                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1789                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1790                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1791                         if (s2 == d) {
1792                                 M_FLTMOVE(s2, REG_FTMP2);
1793                                 s2 = REG_FTMP2;
1794                         }
1795                         M_FLTMOVE(s1, d);
1796                         x86_64_divss_reg_reg(cd, s2, d);
1797                         emit_store(jd, iptr, iptr->dst, d);
1798                         break;
1799
1800                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1801
1802                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1803                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1804                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1805                         if (s2 == d) {
1806                                 M_FLTMOVE(s2, REG_FTMP2);
1807                                 s2 = REG_FTMP2;
1808                         }
1809                         M_FLTMOVE(s1, d);
1810                         x86_64_divsd_reg_reg(cd, s2, d);
1811                         emit_store(jd, iptr, iptr->dst, d);
1812                         break;
1813
1814                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1815
1816                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1817                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1818                         x86_64_cvtsi2ss_reg_reg(cd, s1, d);
1819                         emit_store(jd, iptr, iptr->dst, d);
1820                         break;
1821
1822                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1823
1824                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1825                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1826                         x86_64_cvtsi2sd_reg_reg(cd, s1, d);
1827                         emit_store(jd, iptr, iptr->dst, d);
1828                         break;
1829
1830                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1831
1832                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1833                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1834                         x86_64_cvtsi2ssq_reg_reg(cd, s1, d);
1835                         emit_store(jd, iptr, iptr->dst, d);
1836                         break;
1837
1838                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1839
1840                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1841                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1842                         x86_64_cvtsi2sdq_reg_reg(cd, s1, d);
1843                         emit_store(jd, iptr, iptr->dst, d);
1844                         break;
1845
1846                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1847
1848                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1849                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1850                         x86_64_cvttss2si_reg_reg(cd, s1, d);
1851                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1852                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1853                         x86_64_jcc(cd, X86_64_CC_NE, a);
1854                         M_FLTMOVE(s1, REG_FTMP1);
1855                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_f2i, REG_ITMP2);
1856                         x86_64_call_reg(cd, REG_ITMP2);
1857                         M_INTMOVE(REG_RESULT, d);
1858                         emit_store(jd, iptr, iptr->dst, d);
1859                         break;
1860
1861                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1862
1863                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1864                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1865                         x86_64_cvttsd2si_reg_reg(cd, s1, d);
1866                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1867                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1868                         x86_64_jcc(cd, X86_64_CC_NE, a);
1869                         M_FLTMOVE(s1, REG_FTMP1);
1870                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_d2i, REG_ITMP2);
1871                         x86_64_call_reg(cd, REG_ITMP2);
1872                         M_INTMOVE(REG_RESULT, d);
1873                         emit_store(jd, iptr, iptr->dst, d);
1874                         break;
1875
1876                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1877
1878                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1879                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1880                         x86_64_cvttss2siq_reg_reg(cd, s1, d);
1881                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1882                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1883                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1884                         x86_64_jcc(cd, X86_64_CC_NE, a);
1885                         M_FLTMOVE(s1, REG_FTMP1);
1886                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_f2l, REG_ITMP2);
1887                         x86_64_call_reg(cd, REG_ITMP2);
1888                         M_INTMOVE(REG_RESULT, d);
1889                         emit_store(jd, iptr, iptr->dst, d);
1890                         break;
1891
1892                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1893
1894                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1895                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1896                         x86_64_cvttsd2siq_reg_reg(cd, s1, d);
1897                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1898                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1899                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1900                         x86_64_jcc(cd, X86_64_CC_NE, a);
1901                         M_FLTMOVE(s1, REG_FTMP1);
1902                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_d2l, REG_ITMP2);
1903                         x86_64_call_reg(cd, REG_ITMP2);
1904                         M_INTMOVE(REG_RESULT, d);
1905                         emit_store(jd, iptr, iptr->dst, d);
1906                         break;
1907
1908                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1909
1910                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1911                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1912                         x86_64_cvtss2sd_reg_reg(cd, s1, d);
1913                         emit_store(jd, iptr, iptr->dst, d);
1914                         break;
1915
1916                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1917
1918                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1919                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1920                         x86_64_cvtsd2ss_reg_reg(cd, s1, d);
1921                         emit_store(jd, iptr, iptr->dst, d);
1922                         break;
1923
1924                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1925                                           /* == => 0, < => 1, > => -1 */
1926
1927                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1928                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1929                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1930                         M_CLR(d);
1931                         M_MOV_IMM(1, REG_ITMP1);
1932                         M_MOV_IMM(-1, REG_ITMP2);
1933                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1934                         M_CMOVB(REG_ITMP1, d);
1935                         M_CMOVA(REG_ITMP2, d);
1936                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1937                         emit_store(jd, iptr, iptr->dst, d);
1938                         break;
1939
1940                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1941                                           /* == => 0, < => 1, > => -1 */
1942
1943                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1944                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1945                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1946                         M_CLR(d);
1947                         M_MOV_IMM(1, REG_ITMP1);
1948                         M_MOV_IMM(-1, REG_ITMP2);
1949                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1950                         M_CMOVB(REG_ITMP1, d);
1951                         M_CMOVA(REG_ITMP2, d);
1952                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1953                         emit_store(jd, iptr, iptr->dst, d);
1954                         break;
1955
1956                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1957                                           /* == => 0, < => 1, > => -1 */
1958
1959                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1960                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1961                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1962                         M_CLR(d);
1963                         M_MOV_IMM(1, REG_ITMP1);
1964                         M_MOV_IMM(-1, REG_ITMP2);
1965                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1966                         M_CMOVB(REG_ITMP1, d);
1967                         M_CMOVA(REG_ITMP2, d);
1968                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1969                         emit_store(jd, iptr, iptr->dst, d);
1970                         break;
1971
1972                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1973                                           /* == => 0, < => 1, > => -1 */
1974
1975                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1976                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1977                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1978                         M_CLR(d);
1979                         M_MOV_IMM(1, REG_ITMP1);
1980                         M_MOV_IMM(-1, REG_ITMP2);
1981                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1982                         M_CMOVB(REG_ITMP1, d);
1983                         M_CMOVA(REG_ITMP2, d);
1984                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1985                         emit_store(jd, iptr, iptr->dst, d);
1986                         break;
1987
1988
1989                 /* memory operations **************************************************/
1990
1991                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1992
1993                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1994                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1995                         gen_nullptr_check(s1);
1996                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1997                         emit_store(jd, iptr, iptr->dst, d);
1998                         break;
1999
2000                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2001
2002                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2003                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2004                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2005                         if (iptr->op1 == 0) {
2006                                 gen_nullptr_check(s1);
2007                                 gen_bound_check;
2008                         }
2009                         x86_64_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2010                         emit_store(jd, iptr, iptr->dst, d);
2011                         break;
2012
2013                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2014
2015                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2016                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2017                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2018                         if (iptr->op1 == 0) {
2019                                 gen_nullptr_check(s1);
2020                                 gen_bound_check;
2021                         }
2022                         x86_64_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2023                         emit_store(jd, iptr, iptr->dst, d);
2024                         break;
2025
2026                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2027
2028                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2029                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2030                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2031                         if (iptr->op1 == 0) {
2032                                 gen_nullptr_check(s1);
2033                                 gen_bound_check;
2034                         }
2035                         x86_64_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2036                         emit_store(jd, iptr, iptr->dst, d);
2037                         break;
2038
2039                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
2040
2041                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2042                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2043                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2044                         if (iptr->op1 == 0) {
2045                                 gen_nullptr_check(s1);
2046                                 gen_bound_check;
2047                         }
2048                         x86_64_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
2049                         emit_store(jd, iptr, iptr->dst, d);
2050                         break;
2051
2052                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
2053
2054                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2055                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2056                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2057                         if (iptr->op1 == 0) {
2058                                 gen_nullptr_check(s1);
2059                                 gen_bound_check;
2060                         }
2061                         x86_64_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
2062                         emit_store(jd, iptr, iptr->dst, d);
2063                         break;
2064
2065                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
2066
2067                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2068                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2069                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
2070                         if (iptr->op1 == 0) {
2071                                 gen_nullptr_check(s1);
2072                                 gen_bound_check;
2073                         }
2074                         x86_64_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2075                         emit_store(jd, iptr, iptr->dst, d);
2076                         break;
2077
2078                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2079
2080                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2081                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2082                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
2083                         if (iptr->op1 == 0) {
2084                                 gen_nullptr_check(s1);
2085                                 gen_bound_check;
2086                         }
2087                         x86_64_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2088                         emit_store(jd, iptr, iptr->dst, d);
2089                         break;
2090
2091                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
2092
2093                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2094                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2095                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2096                         if (iptr->op1 == 0) {
2097                                 gen_nullptr_check(s1);
2098                                 gen_bound_check;
2099                         }
2100                         x86_64_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
2101                         emit_store(jd, iptr, iptr->dst, d);
2102                         break;
2103
2104
2105                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2106
2107                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2108                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2109                         if (iptr->op1 == 0) {
2110                                 gen_nullptr_check(s1);
2111                                 gen_bound_check;
2112                         }
2113                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2114                         x86_64_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2115                         break;
2116
2117                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2118
2119                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2120                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2121                         if (iptr->op1 == 0) {
2122                                 gen_nullptr_check(s1);
2123                                 gen_bound_check;
2124                         }
2125                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2126                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2127                         break;
2128
2129                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2130
2131                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2132                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2133                         if (iptr->op1 == 0) {
2134                                 gen_nullptr_check(s1);
2135                                 gen_bound_check;
2136                         }
2137                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2138                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2139                         break;
2140
2141                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2142
2143                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2144                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2145                         if (iptr->op1 == 0) {
2146                                 gen_nullptr_check(s1);
2147                                 gen_bound_check;
2148                         }
2149                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2150                         x86_64_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2151                         break;
2152
2153                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2154
2155                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2156                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2157                         if (iptr->op1 == 0) {
2158                                 gen_nullptr_check(s1);
2159                                 gen_bound_check;
2160                         }
2161                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2162                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2163                         break;
2164
2165                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2166
2167                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2168                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2169                         if (iptr->op1 == 0) {
2170                                 gen_nullptr_check(s1);
2171                                 gen_bound_check;
2172                         }
2173                         s3 = emit_load_s3(jd, iptr, src, REG_FTMP3);
2174                         x86_64_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2175                         break;
2176
2177                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2178
2179                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2180                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2181                         if (iptr->op1 == 0) {
2182                                 gen_nullptr_check(s1);
2183                                 gen_bound_check;
2184                         }
2185                         s3 = emit_load_s3(jd, iptr, src, REG_FTMP3);
2186                         x86_64_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2187                         break;
2188
2189                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2190
2191                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2192                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2193                         if (iptr->op1 == 0) {
2194                                 gen_nullptr_check(s1);
2195                                 gen_bound_check;
2196                         }
2197                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2198
2199                         M_MOV(s1, rd->argintregs[0]);
2200                         M_MOV(s3, rd->argintregs[1]);
2201                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
2202                         M_CALL(REG_ITMP1);
2203                         M_TEST(REG_RESULT);
2204                         M_BEQ(0);
2205                         codegen_add_arraystoreexception_ref(cd, cd->mcodeptr);
2206
2207                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2208                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2209                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2210                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2211                         break;
2212
2213
2214                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2215
2216                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2217                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2218                         if (iptr->op1 == 0) {
2219                                 gen_nullptr_check(s1);
2220                                 gen_bound_check;
2221                         }
2222                         x86_64_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2223                         break;
2224
2225                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2226
2227                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2228                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2229                         if (iptr->op1 == 0) {
2230                                 gen_nullptr_check(s1);
2231                                 gen_bound_check;
2232                         }
2233                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2234                         break;
2235
2236                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2237
2238                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2239                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2240                         if (iptr->op1 == 0) {
2241                                 gen_nullptr_check(s1);
2242                                 gen_bound_check;
2243                         }
2244                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2245                         break;
2246
2247                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2248
2249                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2250                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2251                         if (iptr->op1 == 0) {
2252                                 gen_nullptr_check(s1);
2253                                 gen_bound_check;
2254                         }
2255                         x86_64_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2256                         break;
2257
2258                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2259
2260                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2261                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2262                         if (iptr->op1 == 0) {
2263                                 gen_nullptr_check(s1);
2264                                 gen_bound_check;
2265                         }
2266
2267                         if (IS_IMM32(iptr->val.l)) {
2268                                 x86_64_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2269                         } else {
2270                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2271                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2272                         }
2273                         break;
2274
2275                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2276
2277                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2278                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2279                         if (iptr->op1 == 0) {
2280                                 gen_nullptr_check(s1);
2281                                 gen_bound_check;
2282                         }
2283                         x86_64_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2284                         break;
2285
2286
2287                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2288                                       /* op1 = type, val.a = field address            */
2289
2290                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2291                                 disp = dseg_addaddress(cd, NULL);
2292
2293 /*                              PROFILE_CYCLE_STOP; */
2294
2295                                 codegen_addpatchref(cd, cd->mcodeptr,
2296                                                                         PATCHER_get_putstatic,
2297                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), disp);
2298
2299                                 if (opt_showdisassemble) {
2300                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2301                                 }
2302
2303 /*                              PROFILE_CYCLE_START; */
2304
2305                         } else {
2306                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr);
2307
2308                                 disp = dseg_addaddress(cd, &(fi->value));
2309
2310                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2311                                         PROFILE_CYCLE_STOP;
2312
2313                                         codegen_addpatchref(cd, cd->mcodeptr,
2314                                                                                 PATCHER_clinit, fi->class, 0);
2315
2316                                         if (opt_showdisassemble) {
2317                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2318                                         }
2319
2320                                         PROFILE_CYCLE_START;
2321                                 }
2322                         }
2323
2324                         /* This approach is much faster than moving the field
2325                            address inline into a register. */
2326
2327                         M_ALD(REG_ITMP2, RIP, -(((ptrint) cd->mcodeptr + 7) -
2328                                                                         (ptrint) cd->mcodebase) + disp);
2329
2330                         switch (iptr->op1) {
2331                         case TYPE_INT:
2332                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2333                                 M_ILD(d, REG_ITMP2, 0);
2334                                 emit_store(jd, iptr, iptr->dst, d);
2335                                 break;
2336                         case TYPE_LNG:
2337                         case TYPE_ADR:
2338                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2339                                 M_LLD(d, REG_ITMP2, 0);
2340                                 emit_store(jd, iptr, iptr->dst, d);
2341                                 break;
2342                         case TYPE_FLT:
2343                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2344                                 x86_64_movss_membase_reg(cd, REG_ITMP2, 0, d);
2345                                 emit_store(jd, iptr, iptr->dst, d);
2346                                 break;
2347                         case TYPE_DBL:
2348                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2349                                 x86_64_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2350                                 emit_store(jd, iptr, iptr->dst, d);
2351                                 break;
2352                         }
2353                         break;
2354
2355                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2356                                       /* op1 = type, val.a = field address            */
2357
2358                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2359                                 disp = dseg_addaddress(cd, NULL);
2360
2361 /*                              PROFILE_CYCLE_STOP; */
2362
2363                                 codegen_addpatchref(cd, cd->mcodeptr,
2364                                                                         PATCHER_get_putstatic,
2365                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), disp);
2366
2367                                 if (opt_showdisassemble) {
2368                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2369                                 }
2370
2371 /*                              PROFILE_CYCLE_START; */
2372
2373                         } else {
2374                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr);
2375
2376                                 disp = dseg_addaddress(cd, &(fi->value));
2377
2378                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2379                                         PROFILE_CYCLE_STOP;
2380
2381                                         codegen_addpatchref(cd, cd->mcodeptr,
2382                                                                                 PATCHER_clinit, fi->class, 0);
2383
2384                                         if (opt_showdisassemble) {
2385                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2386                                         }
2387
2388                                         PROFILE_CYCLE_START;
2389                                 }
2390                         }
2391
2392                         /* This approach is much faster than moving the field
2393                            address inline into a register. */
2394
2395                         M_ALD(REG_ITMP2, RIP, -(((ptrint) cd->mcodeptr + 7) -
2396                                                                         (ptrint) cd->mcodebase) + disp);
2397
2398                         switch (iptr->op1) {
2399                         case TYPE_INT:
2400                                 s2 = emit_load_s2(jd, iptr, src, REG_ITMP1);
2401                                 M_IST(s2, REG_ITMP2, 0);
2402                                 break;
2403                         case TYPE_LNG:
2404                         case TYPE_ADR:
2405                                 s2 = emit_load_s2(jd, iptr, src, REG_ITMP1);
2406                                 M_LST(s2, REG_ITMP2, 0);
2407                                 break;
2408                         case TYPE_FLT:
2409                                 s2 = emit_load_s2(jd, iptr, src, REG_FTMP1);
2410                                 x86_64_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2411                                 break;
2412                         case TYPE_DBL:
2413                                 s2 = emit_load_s2(jd, iptr, src, REG_FTMP1);
2414                                 x86_64_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2415                                 break;
2416                         }
2417                         break;
2418
2419                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2420                                           /* val = value (in current instruction)     */
2421                                           /* op1 = type, val.a = field address (in    */
2422                                           /* following NOP)                           */
2423
2424                         if (INSTRUCTION_IS_UNRESOLVED(iptr + 1)) {
2425                                 disp = dseg_addaddress(cd, NULL);
2426
2427 /*                              PROFILE_CYCLE_STOP; */
2428
2429                                 codegen_addpatchref(cd, cd->mcodeptr,
2430                                                                         PATCHER_get_putstatic,
2431                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr + 1), disp);
2432
2433                                 if (opt_showdisassemble) {
2434                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2435                                 }
2436
2437 /*                              PROFILE_CYCLE_START; */
2438
2439                         } else {
2440                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr + 1);
2441
2442                                 disp = dseg_addaddress(cd, &(fi->value));
2443
2444                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2445                                         PROFILE_CYCLE_STOP;
2446
2447                                         codegen_addpatchref(cd, cd->mcodeptr,
2448                                                                                 PATCHER_clinit, fi->class, 0);
2449
2450                                         if (opt_showdisassemble) {
2451                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2452                                         }
2453
2454                                         PROFILE_CYCLE_START;
2455                                 }
2456                         }
2457
2458                         /* This approach is much faster than moving the field
2459                            address inline into a register. */
2460
2461                         M_ALD(REG_ITMP1, RIP, -(((ptrint) cd->mcodeptr + 7) -
2462                                                                         (ptrint) cd->mcodebase) + disp);
2463
2464                         switch (iptr->op1) {
2465                         case TYPE_INT:
2466                         case TYPE_FLT:
2467                                 M_IST_IMM(iptr->val.i, REG_ITMP1, 0);
2468                                 break;
2469                         case TYPE_LNG:
2470                         case TYPE_ADR:
2471                         case TYPE_DBL:
2472                                 if (IS_IMM32(iptr->val.l)) {
2473                                         M_LST_IMM32(iptr->val.l, REG_ITMP1, 0);
2474                                 } else {
2475                                         M_IST_IMM(iptr->val.l, REG_ITMP1, 0);
2476                                         M_IST_IMM(iptr->val.l >> 32, REG_ITMP1, 4);
2477                                 }
2478                                 break;
2479                         }
2480                         break;
2481
2482                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2483                                       /* op1 = type, val.i = field offset             */
2484
2485                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2486                         gen_nullptr_check(s1);
2487
2488                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2489 /*                              PROFILE_CYCLE_STOP; */
2490
2491                                 codegen_addpatchref(cd, cd->mcodeptr,
2492                                                                         PATCHER_get_putfield,
2493                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), 0);
2494
2495                                 if (opt_showdisassemble) {
2496                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2497                                 }
2498
2499 /*                              PROFILE_CYCLE_START; */
2500
2501                                 disp = 0;
2502
2503                         } else {
2504                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr)->offset;
2505                         }
2506
2507                         switch (iptr->op1) {
2508                         case TYPE_INT:
2509                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2510                                 M_ILD32(d, s1, disp);
2511                                 emit_store(jd, iptr, iptr->dst, d);
2512                                 break;
2513                         case TYPE_LNG:
2514                         case TYPE_ADR:
2515                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2516                                 M_LLD32(d, s1, disp);
2517                                 emit_store(jd, iptr, iptr->dst, d);
2518                                 break;
2519                         case TYPE_FLT:
2520                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2521                                 x86_64_movss_membase32_reg(cd, s1, disp, d);
2522                                 emit_store(jd, iptr, iptr->dst, d);
2523                                 break;
2524                         case TYPE_DBL:
2525                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2526                                 x86_64_movsd_membase32_reg(cd, s1, disp, d);
2527                                 emit_store(jd, iptr, iptr->dst, d);
2528                                 break;
2529                         }
2530                         break;
2531
2532                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2533                                       /* op1 = type, val.i = field offset             */
2534
2535                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2536                         gen_nullptr_check(s1);
2537
2538                         if (IS_INT_LNG_TYPE(iptr->op1)) {
2539                                 s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2540                         } else {
2541                                 s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
2542                         }
2543
2544                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2545 /*                              PROFILE_CYCLE_STOP; */
2546
2547                                 codegen_addpatchref(cd, cd->mcodeptr,
2548                                                                         PATCHER_get_putfield,
2549                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), 0);
2550
2551                                 if (opt_showdisassemble) {
2552                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2553                                 }
2554
2555 /*                              PROFILE_CYCLE_START; */
2556
2557                                 disp = 0;
2558
2559                         } else {
2560                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr)->offset;
2561                         }
2562
2563                         switch (iptr->op1) {
2564                         case TYPE_INT:
2565                                 M_IST32(s2, s1, disp);
2566                                 break;
2567                         case TYPE_LNG:
2568                         case TYPE_ADR:
2569                                 M_LST32(s2, s1, disp);
2570                                 break;
2571                         case TYPE_FLT:
2572                                 x86_64_movss_reg_membase32(cd, s2, s1, disp);
2573                                 break;
2574                         case TYPE_DBL:
2575                                 x86_64_movsd_reg_membase32(cd, s2, s1, disp);
2576                                 break;
2577                         }
2578                         break;
2579
2580                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2581                                           /* val = value (in current instruction)     */
2582                                           /* op1 = type, val.a = field address (in    */
2583                                           /* following NOP)                           */
2584
2585                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2586                         gen_nullptr_check(s1);
2587
2588                         if (INSTRUCTION_IS_UNRESOLVED(iptr + 1)) {
2589 /*                              PROFILE_CYCLE_STOP; */
2590
2591                                 codegen_addpatchref(cd, cd->mcodeptr,
2592                                                                         PATCHER_putfieldconst,
2593                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr + 1), 0);
2594
2595                                 if (opt_showdisassemble) {
2596                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2597                                 }
2598
2599 /*                              PROFILE_CYCLE_START; */
2600
2601                                 disp = 0;
2602
2603                         } else {
2604                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr + 1)->offset;
2605                         }
2606
2607                         switch (iptr->op1) {
2608                         case TYPE_INT:
2609                         case TYPE_FLT:
2610                                 M_IST32_IMM(iptr->val.i, s1, disp);
2611                                 break;
2612                         case TYPE_LNG:
2613                         case TYPE_ADR:
2614                         case TYPE_DBL:
2615                                 M_IST32_IMM(iptr->val.l, s1, disp);
2616                                 M_IST32_IMM(iptr->val.l >> 32, s1, disp + 4);
2617                                 break;
2618                         }
2619                         break;
2620
2621
2622                 /* branch operations **************************************************/
2623
2624                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2625
2626                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2627                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2628
2629                         PROFILE_CYCLE_STOP;
2630
2631 #ifdef ENABLE_VERIFIER
2632                         if (iptr->val.a) {
2633                                 codegen_addpatchref(cd, cd->mcodeptr,
2634                                                                         PATCHER_athrow_areturn,
2635                                                                         (unresolved_class *) iptr->val.a, 0);
2636
2637                                 if (opt_showdisassemble) {
2638                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2639                                 }
2640                         }
2641 #endif /* ENABLE_VERIFIER */
2642
2643                         M_CALL_IMM(0);                            /* passing exception pc */
2644                         M_POP(REG_ITMP2_XPC);
2645
2646                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2647                         M_JMP(REG_ITMP3);
2648                         break;
2649
2650                 case ICMD_GOTO:         /* ... ==> ...                                */
2651                                         /* op1 = target JavaVM pc                     */
2652
2653                         M_JMP_IMM(0);
2654                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2655                         break;
2656
2657                 case ICMD_JSR:          /* ... ==> ...                                */
2658                                         /* op1 = target JavaVM pc                     */
2659
2660                         M_CALL_IMM(0);
2661                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2662                         break;
2663
2664                 case ICMD_RET:          /* ... ==> ...                                */
2665                                         /* op1 = local variable                       */
2666
2667                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2668                         if (var->flags & INMEMORY) {
2669                                 M_ALD(REG_ITMP1, REG_SP, var->regoff * 8);
2670                                 M_JMP(REG_ITMP1);
2671                         } else
2672                                 M_JMP(var->regoff);
2673                         break;
2674
2675                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2676                                         /* op1 = target JavaVM pc                     */
2677
2678                         if (src->flags & INMEMORY)
2679                                 M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
2680                         else
2681                                 M_TEST(src->regoff);
2682
2683                         /* If the conditional branch is part of an if-converted
2684                            block, don't generate the actual branch. */
2685
2686                         if ((iptr->opc & ICMD_CONDITION_MASK) == 0) {
2687                                 M_BEQ(0);
2688                                 codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2689                         }
2690                         break;
2691
2692                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2693                                         /* op1 = target JavaVM pc                     */
2694
2695                         if (src->flags & INMEMORY)
2696                                 M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
2697                         else
2698                                 M_TEST(src->regoff);
2699
2700                         /* If the conditional branch is part of an if-converted
2701                            block, don't generate the actual branch. */
2702
2703                         if ((iptr->opc & ICMD_CONDITION_MASK) == 0) {
2704                                 M_BNE(0);
2705                                 codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2706                         }
2707                         break;
2708
2709                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2710                                         /* op1 = target JavaVM pc, val.i = constant   */
2711
2712                         x86_64_emit_ifcc(cd, X86_64_CC_E, src, iptr);
2713                         break;
2714
2715                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2716                                         /* op1 = target JavaVM pc, val.i = constant   */
2717
2718                         x86_64_emit_ifcc(cd, X86_64_CC_L, src, iptr);
2719                         break;
2720
2721                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2722                                         /* op1 = target JavaVM pc, val.i = constant   */
2723
2724                         x86_64_emit_ifcc(cd, X86_64_CC_LE, src, iptr);
2725                         break;
2726
2727                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2728                                         /* op1 = target JavaVM pc, val.i = constant   */
2729
2730                         x86_64_emit_ifcc(cd, X86_64_CC_NE, src, iptr);
2731                         break;
2732
2733                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2734                                         /* op1 = target JavaVM pc, val.i = constant   */
2735
2736                         x86_64_emit_ifcc(cd, X86_64_CC_G, src, iptr);
2737                         break;
2738
2739                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2740                                         /* op1 = target JavaVM pc, val.i = constant   */
2741
2742                         x86_64_emit_ifcc(cd, X86_64_CC_GE, src, iptr);
2743                         break;
2744
2745                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2746                                         /* op1 = target JavaVM pc, val.l = constant   */
2747
2748                         x86_64_emit_if_lcc(cd, X86_64_CC_E, src, iptr);
2749                         break;
2750
2751                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2752                                         /* op1 = target JavaVM pc, val.l = constant   */
2753
2754                         x86_64_emit_if_lcc(cd, X86_64_CC_L, src, iptr);
2755                         break;
2756
2757                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2758                                         /* op1 = target JavaVM pc, val.l = constant   */
2759
2760                         x86_64_emit_if_lcc(cd, X86_64_CC_LE, src, iptr);
2761                         break;
2762
2763                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2764                                         /* op1 = target JavaVM pc, val.l = constant   */
2765
2766                         x86_64_emit_if_lcc(cd, X86_64_CC_NE, src, iptr);
2767                         break;
2768
2769                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2770                                         /* op1 = target JavaVM pc, val.l = constant   */
2771
2772                         x86_64_emit_if_lcc(cd, X86_64_CC_G, src, iptr);
2773                         break;
2774
2775                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2776                                         /* op1 = target JavaVM pc, val.l = constant   */
2777
2778                         x86_64_emit_if_lcc(cd, X86_64_CC_GE, src, iptr);
2779                         break;
2780
2781                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2782                                         /* op1 = target JavaVM pc                     */
2783
2784                         x86_64_emit_if_icmpcc(cd, X86_64_CC_E, src, iptr);
2785                         break;
2786
2787                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2788                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2789
2790                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_E, src, iptr);
2791                         break;
2792
2793                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2794                                         /* op1 = target JavaVM pc                     */
2795
2796                         x86_64_emit_if_icmpcc(cd, X86_64_CC_NE, src, iptr);
2797                         break;
2798
2799                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2800                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2801
2802                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_NE, src, iptr);
2803                         break;
2804
2805                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2806                                         /* op1 = target JavaVM pc                     */
2807
2808                         x86_64_emit_if_icmpcc(cd, X86_64_CC_L, src, iptr);
2809                         break;
2810
2811                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2812                                     /* op1 = target JavaVM pc                     */
2813
2814                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_L, src, iptr);
2815                         break;
2816
2817                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2818                                         /* op1 = target JavaVM pc                     */
2819
2820                         x86_64_emit_if_icmpcc(cd, X86_64_CC_G, src, iptr);
2821                         break;
2822
2823                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2824                                 /* op1 = target JavaVM pc                     */
2825
2826                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_G, src, iptr);
2827                         break;
2828
2829                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2830                                         /* op1 = target JavaVM pc                     */
2831
2832                         x86_64_emit_if_icmpcc(cd, X86_64_CC_LE, src, iptr);
2833                         break;
2834
2835                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2836                                         /* op1 = target JavaVM pc                     */
2837
2838                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_LE, src, iptr);
2839                         break;
2840
2841                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2842                                         /* op1 = target JavaVM pc                     */
2843
2844                         x86_64_emit_if_icmpcc(cd, X86_64_CC_GE, src, iptr);
2845                         break;
2846
2847                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2848                                     /* op1 = target JavaVM pc                     */
2849
2850                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_GE, src, iptr);
2851                         break;
2852
2853                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2854
2855                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2856                         break;
2857
2858                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2859                 case ICMD_IFNE_ICONST:  /* val.i = constant                           */
2860                 case ICMD_IFLT_ICONST:
2861                 case ICMD_IFGE_ICONST:
2862                 case ICMD_IFGT_ICONST:
2863                 case ICMD_IFLE_ICONST:
2864
2865                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2866                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2867                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2868                                 if (s1 == d) {
2869                                         M_INTMOVE(s1, REG_ITMP1);
2870                                         s1 = REG_ITMP1;
2871                                 }
2872                                 if (iptr[1].val.i == 0)
2873                                         M_CLR(d);
2874                                 else
2875                                         M_IMOV_IMM(iptr[1].val.i, d);
2876                         }
2877                         if (iptr->val.i == 0)
2878                                 M_CLR(REG_ITMP2);
2879                         else
2880                                 M_IMOV_IMM(iptr->val.i, REG_ITMP2);
2881                         M_ITEST(s1);
2882
2883                         switch (iptr->opc) {
2884                         case ICMD_IFEQ_ICONST:
2885                                 M_CMOVEQ(REG_ITMP2, d);
2886                                 break;
2887                         case ICMD_IFNE_ICONST:
2888                                 M_CMOVNE(REG_ITMP2, d);
2889                                 break;
2890                         case ICMD_IFLT_ICONST:
2891                                 M_CMOVLT(REG_ITMP2, d);
2892                                 break;
2893                         case ICMD_IFGE_ICONST:
2894                                 M_CMOVGE(REG_ITMP2, d);
2895                                 break;
2896                         case ICMD_IFGT_ICONST:
2897                                 M_CMOVGT(REG_ITMP2, d);
2898                                 break;
2899                         case ICMD_IFLE_ICONST:
2900                                 M_CMOVLE(REG_ITMP2, d);
2901                                 break;
2902                         }
2903
2904                         emit_store(jd, iptr, iptr->dst, d);
2905                         break;
2906
2907
2908                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2909                 case ICMD_LRETURN:
2910
2911                         s1 = emit_load_s1(jd, iptr, src, REG_RESULT);
2912                         M_INTMOVE(s1, REG_RESULT);
2913                         goto nowperformreturn;
2914
2915                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2916
2917                         s1 = emit_load_s1(jd, iptr, src, REG_RESULT);
2918                         M_INTMOVE(s1, REG_RESULT);
2919
2920 #ifdef ENABLE_VERIFIER
2921                         if (iptr->val.a) {
2922                                 PROFILE_CYCLE_STOP;
2923
2924                                 codegen_addpatchref(cd, cd->mcodeptr,
2925                                                                         PATCHER_athrow_areturn,
2926                                                                         (unresolved_class *) iptr->val.a, 0);
2927
2928                                 if (opt_showdisassemble) {
2929                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2930                                 }
2931
2932                                 PROFILE_CYCLE_START;
2933                         }
2934 #endif /* ENABLE_VERIFIER */
2935                         goto nowperformreturn;
2936
2937                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2938                 case ICMD_DRETURN:
2939
2940                         s1 = emit_load_s1(jd, iptr, src, REG_FRESULT);
2941                         M_FLTMOVE(s1, REG_FRESULT);
2942                         goto nowperformreturn;
2943
2944                 case ICMD_RETURN:      /* ...  ==> ...                                */
2945
2946 nowperformreturn:
2947                         {
2948                         s4 i, p;
2949
2950                         p = stackframesize;
2951
2952 #if !defined(NDEBUG)
2953                         /* generate call trace */
2954
2955                         if (opt_verbosecall) {
2956                                 x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
2957
2958                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2959                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2960
2961                                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
2962                                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2963                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2964                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2965
2966                                 M_MOV_IMM(builtin_displaymethodstop, REG_ITMP1);
2967                                 M_CALL(REG_ITMP1);
2968
2969                                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2970                                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2971
2972                                 x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
2973                         }
2974 #endif /* !defined(NDEBUG) */
2975
2976 #if defined(USE_THREADS)
2977                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2978                                 M_ALD(rd->argintregs[0], REG_SP, rd->memuse * 8);
2979
2980                                 /* we need to save the proper return value */
2981                                 switch (iptr->opc) {
2982                                 case ICMD_IRETURN:
2983                                 case ICMD_ARETURN:
2984                                 case ICMD_LRETURN:
2985                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2986                                         break;
2987                                 case ICMD_FRETURN:
2988                                 case ICMD_DRETURN:
2989                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2990                                         break;
2991                                 }
2992
2993                                 M_MOV_IMM(builtin_monitorexit, REG_ITMP1);
2994                                 M_CALL(REG_ITMP1);
2995
2996                                 /* and now restore the proper return value */
2997                                 switch (iptr->opc) {
2998                                 case ICMD_IRETURN:
2999                                 case ICMD_ARETURN:
3000                                 case ICMD_LRETURN:
3001                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
3002                                         break;
3003                                 case ICMD_FRETURN:
3004                                 case ICMD_DRETURN:
3005                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
3006                                         break;
3007                                 }
3008                         }
3009 #endif
3010
3011                         /* restore saved registers */
3012
3013                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
3014                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
3015                         }
3016                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
3017                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
3018                         }
3019
3020                         /* deallocate stack */
3021
3022                         if (stackframesize)
3023                                 M_AADD_IMM(stackframesize * 8, REG_SP);
3024
3025                         /* generate method profiling code */
3026
3027                         PROFILE_CYCLE_STOP;
3028
3029                         M_RET;
3030                         }
3031                         break;
3032
3033
3034                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
3035                         {
3036                                 s4 i, l, *s4ptr;
3037                                 void **tptr;
3038
3039                                 tptr = (void **) iptr->target;
3040
3041                                 s4ptr = iptr->val.a;
3042                                 l = s4ptr[1];                          /* low     */
3043                                 i = s4ptr[2];                          /* high    */
3044
3045                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3046                                 M_INTMOVE(s1, REG_ITMP1);
3047                                 if (l != 0) {
3048                                         x86_64_alul_imm_reg(cd, X86_64_SUB, l, REG_ITMP1);
3049                                 }
3050                                 i = i - l + 1;
3051
3052                 /* range check */
3053                                 x86_64_alul_imm_reg(cd, X86_64_CMP, i - 1, REG_ITMP1);
3054                                 x86_64_jcc(cd, X86_64_CC_A, 0);
3055
3056                 /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[0]), cd->mcodeptr); */
3057                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3058
3059                                 /* build jump table top down and use address of lowest entry */
3060
3061                 /* s4ptr += 3 + i; */
3062                                 tptr += i;
3063
3064                                 while (--i >= 0) {
3065                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
3066                                         --tptr;
3067                                 }
3068
3069                                 /* length of dataseg after last dseg_addtarget is used by load */
3070
3071                                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2);
3072                                 dseg_adddata(cd, cd->mcodeptr);
3073                                 x86_64_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
3074                                 x86_64_jmp_reg(cd, REG_ITMP1);
3075                         }
3076                         break;
3077
3078
3079                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
3080                         {
3081                                 s4 i, l, val, *s4ptr;
3082                                 void **tptr;
3083
3084                                 tptr = (void **) iptr->target;
3085
3086                                 s4ptr = iptr->val.a;
3087                                 l = s4ptr[0];                          /* default  */
3088                                 i = s4ptr[1];                          /* count    */
3089
3090                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
3091                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);    /* reg compare should always be faster */
3092                                 while (--i >= 0) {
3093                                         s4ptr += 2;
3094                                         ++tptr;
3095
3096                                         val = s4ptr[0];
3097                                         x86_64_alul_imm_reg(cd, X86_64_CMP, val, s1);
3098                                         x86_64_jcc(cd, X86_64_CC_E, 0);
3099                                         codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3100                                 }
3101
3102                                 x86_64_jmp_imm(cd, 0);
3103
3104                                 tptr = (void **) iptr->target;
3105                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3106                         }
3107                         break;
3108
3109
3110                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
3111                                         /* op1 = arg count val.a = builtintable entry */
3112
3113                         bte = iptr->val.a;
3114                         md = bte->md;
3115                         goto gen_method;
3116
3117                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
3118                                         /* op1 = arg count, val.a = method pointer    */
3119
3120                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
3121                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
3122                 case ICMD_INVOKEINTERFACE:
3123
3124                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3125                                 md = INSTRUCTION_UNRESOLVED_METHOD(iptr)->methodref->parseddesc.md;
3126                                 lm = NULL;
3127                         }
3128                         else {
3129                                 lm = INSTRUCTION_RESOLVED_METHODINFO(iptr);
3130                                 md = lm->parseddesc;
3131                         }
3132
3133 gen_method:
3134                         s3 = md->paramcount;
3135
3136                         MCODECHECK((20 * s3) + 128);
3137
3138                         /* copy arguments to registers or stack location */
3139
3140                         for (s3 = s3 - 1; s3 >= 0; s3--, src = src->prev) {
3141                                 if (src->varkind == ARGVAR)
3142                                         continue;
3143                                 if (IS_INT_LNG_TYPE(src->type)) {
3144                                         if (!md->params[s3].inmemory) {
3145                                                 s1 = rd->argintregs[md->params[s3].regoff];
3146                                                 d = emit_load_s1(jd, iptr, src, s1);
3147                                                 M_INTMOVE(d, s1);
3148                                         } else {
3149                                                 d = emit_load_s1(jd, iptr, src, REG_ITMP1);
3150                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
3151                                         }
3152
3153                                 } else {
3154                                         if (!md->params[s3].inmemory) {
3155                                                 s1 = rd->argfltregs[md->params[s3].regoff];
3156                                                 d = emit_load_s1(jd, iptr, src, s1);
3157                                                 M_FLTMOVE(d, s1);
3158                                         } else {
3159                                                 d = emit_load_s1(jd, iptr, src, REG_FTMP1);
3160                                                 M_DST(d, REG_SP, md->params[s3].regoff * 8);
3161                                         }
3162                                 }
3163                         }
3164
3165                         /* generate method profiling code */
3166
3167                         PROFILE_CYCLE_STOP;
3168
3169                         switch (iptr->opc) {
3170                         case ICMD_BUILTIN:
3171                                 a = (ptrint) bte->fp;
3172                                 d = md->returntype.type;
3173
3174                                 M_MOV_IMM(a, REG_ITMP1);
3175                                 M_CALL(REG_ITMP1);
3176
3177                                 /* if op1 == true, we need to check for an exception */
3178
3179                                 if (iptr->op1 == true) {
3180                                         M_TEST(REG_RESULT);
3181                                         M_BEQ(0);
3182                                         codegen_add_fillinstacktrace_ref(cd, cd->mcodeptr);
3183                                 }
3184                                 break;
3185
3186                         case ICMD_INVOKESPECIAL:
3187                                 M_TEST(rd->argintregs[0]);
3188                                 M_BEQ(0);
3189                                 codegen_add_nullpointerexception_ref(cd, cd->mcodeptr);
3190
3191                                 /* first argument contains pointer */
3192 /*                              gen_nullptr_check(rd->argintregs[0]); */
3193
3194                                 /* access memory for hardware nullptr */
3195 /*                              x86_64_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); */
3196
3197                                 /* fall through */
3198
3199                         case ICMD_INVOKESTATIC:
3200                                 if (lm == NULL) {
3201                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3202
3203                                         codegen_addpatchref(cd, cd->mcodeptr,
3204                                                                                 PATCHER_invokestatic_special, um, 0);
3205
3206                                         if (opt_showdisassemble) {
3207                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3208                                         }
3209
3210                                         a = 0;
3211                                         d = um->methodref->parseddesc.md->returntype.type;
3212
3213                                 } else {
3214                                         a = (ptrint) lm->stubroutine;
3215                                         d = lm->parseddesc->returntype.type;
3216                                 }
3217
3218                                 M_MOV_IMM(a, REG_ITMP2);
3219                                 M_CALL(REG_ITMP2);
3220                                 break;
3221
3222                         case ICMD_INVOKEVIRTUAL:
3223                                 gen_nullptr_check(rd->argintregs[0]);
3224
3225                                 if (lm == NULL) {
3226                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3227
3228                                         codegen_addpatchref(cd, cd->mcodeptr,
3229                                                                                 PATCHER_invokevirtual, um, 0);
3230
3231                                         if (opt_showdisassemble) {
3232                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3233                                         }
3234
3235                                         s1 = 0;
3236                                         d = um->methodref->parseddesc.md->returntype.type;
3237
3238                                 } else {
3239                                         s1 = OFFSET(vftbl_t, table[0]) +
3240                                                 sizeof(methodptr) * lm->vftblindex;
3241                                         d = lm->parseddesc->returntype.type;
3242                                 }
3243
3244                                 x86_64_mov_membase_reg(cd, rd->argintregs[0],
3245                                                                            OFFSET(java_objectheader, vftbl),
3246                                                                            REG_ITMP2);
3247                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP1);
3248                                 M_CALL(REG_ITMP1);
3249                                 break;
3250
3251                         case ICMD_INVOKEINTERFACE:
3252                                 gen_nullptr_check(rd->argintregs[0]);
3253
3254                                 if (lm == NULL) {
3255                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3256
3257                                         codegen_addpatchref(cd, cd->mcodeptr,
3258                                                                                 PATCHER_invokeinterface, um, 0);
3259
3260                                         if (opt_showdisassemble) {
3261                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3262                                         }
3263
3264                                         s1 = 0;
3265                                         s2 = 0;
3266                                         d = um->methodref->parseddesc.md->returntype.type;
3267
3268                                 } else {
3269                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3270                                                 sizeof(methodptr) * lm->class->index;
3271
3272                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3273
3274                                         d = lm->parseddesc->returntype.type;
3275                                 }
3276
3277                                 M_ALD(REG_ITMP2, rd->argintregs[0],
3278                                           OFFSET(java_objectheader, vftbl));
3279                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP2);
3280                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s2, REG_ITMP1);
3281                                 M_CALL(REG_ITMP1);
3282                                 break;
3283                         }
3284
3285                         /* generate method profiling code */
3286
3287                         PROFILE_CYCLE_START;
3288
3289                         /* d contains return type */
3290
3291                         if (d != TYPE_VOID) {
3292                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3293                                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_RESULT);
3294                                         M_INTMOVE(REG_RESULT, s1);
3295                                         emit_store(jd, iptr, iptr->dst, s1);
3296                                 } else {
3297                                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FRESULT);
3298                                         M_FLTMOVE(REG_FRESULT, s1);
3299                                         emit_store(jd, iptr, iptr->dst, s1);
3300                                 }
3301                         }
3302                         break;
3303
3304
3305                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3306
3307                                       /* op1:   0 == array, 1 == class                */
3308                                       /* val.a: (classinfo *) superclass              */
3309
3310                         /*  superclass is an interface:
3311                          *
3312                          *  OK if ((sub == NULL) ||
3313                          *         (sub->vftbl->interfacetablelength > super->index) &&
3314                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3315                          *
3316                          *  superclass is a class:
3317                          *
3318                          *  OK if ((sub == NULL) || (0
3319                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3320                          *         super->vftbl->diffval));
3321                          */
3322
3323                         if (iptr->op1 == 1) {
3324                                 /* object type cast-check */
3325
3326                                 classinfo *super;
3327                                 vftbl_t   *supervftbl;
3328                                 s4         superindex;
3329
3330                                 super = (classinfo *) iptr->val.a;
3331
3332                                 if (!super) {
3333                                         superindex = 0;
3334                                         supervftbl = NULL;
3335
3336                                 } else {
3337                                         superindex = super->index;
3338                                         supervftbl = super->vftbl;
3339                                 }
3340
3341 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3342                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3343 #endif
3344                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3345
3346                                 /* calculate interface checkcast code size */
3347
3348                                 s2 = 3; /* mov_membase_reg */
3349                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3350
3351                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3352                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3353                                         3 /* test */ + 6 /* jcc */;
3354
3355                                 if (!super)
3356                                         s2 += (opt_showdisassemble ? 5 : 0);
3357
3358                                 /* calculate class checkcast code size */
3359
3360                                 s3 = 3; /* mov_membase_reg */
3361                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3362                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3363
3364 #if 0
3365                                 if (s1 != REG_ITMP1) {
3366                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3367                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3368                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3369                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3370                                         a += 3;    /* sub */
3371
3372                                 } else
3373 #endif
3374                                         {
3375                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3376                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3377                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3378                                         }
3379
3380                                 s3 += 3 /* cmp */ + 6 /* jcc */;
3381
3382                                 if (!super)
3383                                         s3 += (opt_showdisassemble ? 5 : 0);
3384
3385                                 /* if class is not resolved, check which code to call */
3386
3387                                 if (!super) {
3388                                         M_TEST(s1);
3389                                         M_BEQ(6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3390
3391                                         codegen_addpatchref(cd, cd->mcodeptr,
3392                                                                                 PATCHER_checkcast_instanceof_flags,
3393                                                                                 (constant_classref *) iptr->target, 0);
3394
3395                                         if (opt_showdisassemble) {
3396                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3397                                         }
3398
3399                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
3400                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
3401                                         M_BEQ(s2 + 5);
3402                                 }
3403
3404                                 /* interface checkcast code */
3405
3406                                 if (!super || (super->flags & ACC_INTERFACE)) {
3407                                         if (super) {
3408                                                 M_TEST(s1);
3409                                                 M_BEQ(s2);
3410                                         }
3411
3412                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3413
3414                                         if (!super) {
3415                                                 codegen_addpatchref(cd, cd->mcodeptr,
3416                                                                                         PATCHER_checkcast_instanceof_interface,
3417                                                                                         (constant_classref *) iptr->target, 0);
3418
3419                                                 if (opt_showdisassemble) {
3420                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3421                                                 }
3422                                         }
3423
3424                                         x86_64_movl_membase32_reg(cd, REG_ITMP2,
3425                                                                                           OFFSET(vftbl_t, interfacetablelength),
3426                                                                                           REG_ITMP3);
3427                                         /* XXX TWISTI: should this be int arithmetic? */
3428                                         M_LSUB_IMM32(superindex, REG_ITMP3);
3429                                         M_TEST(REG_ITMP3);
3430                                         M_BLE(0);
3431                                         codegen_add_classcastexception_ref(cd, cd->mcodeptr);
3432                                         x86_64_mov_membase32_reg(cd, REG_ITMP2,
3433                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
3434                                                                                          superindex * sizeof(methodptr*),
3435                                                                                          REG_ITMP3);
3436                                         M_TEST(REG_ITMP3);
3437                                         M_BEQ(0);
3438                                         codegen_add_classcastexception_ref(cd, cd->mcodeptr);
3439
3440                                         if (!super)
3441                                                 M_JMP_IMM(s3);
3442                                 }
3443
3444                                 /* class checkcast code */
3445
3446                                 if (!super || !(super->flags & ACC_INTERFACE)) {
3447                                         if (super) {
3448                                                 M_TEST(s1);
3449                                                 M_BEQ(s3);
3450                                         }
3451
3452                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3453
3454                                         if (!super) {
3455                                                 codegen_addpatchref(cd, cd->mcodeptr,
3456                                                                                         PATCHER_checkcast_class,
3457                                                                                         (constant_classref *) iptr->target,
3458                                                                                         0);
3459
3460                                                 if (opt_showdisassemble) {
3461                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3462                                                 }
3463                                         }
3464
3465                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3466 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3467                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3468 #endif
3469                                         x86_64_movl_membase32_reg(cd, REG_ITMP2,
3470                                                                                           OFFSET(vftbl_t, baseval),
3471                                                                                           REG_ITMP2);
3472                                         /*                                      if (s1 != REG_ITMP1) { */
3473                                         /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3474                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
3475                                         /*                                                                                              REG_ITMP1); */
3476                                         /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3477                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3478                                         /*                                                                                              REG_ITMP3); */
3479                                         /*  #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
3480                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3481                                         /*  #endif */
3482                                         /*                                              x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP1, REG_ITMP2); */
3483
3484                                         /*                                      } else { */
3485                                         x86_64_movl_membase32_reg(cd, REG_ITMP3,
3486                                                                                           OFFSET(vftbl_t, baseval),
3487                                                                                           REG_ITMP3);
3488                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3489                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3490                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3491                                         /*                                      } */
3492 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3493                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3494 #endif
3495                                         M_CMP(REG_ITMP3, REG_ITMP2);
3496                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3497                                         codegen_add_classcastexception_ref(cd, cd->mcodeptr);
3498                                 }
3499                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
3500
3501                         } else {
3502                                 /* array type cast-check */
3503
3504                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3505                                 M_INTMOVE(s1, rd->argintregs[0]);
3506
3507                                 if (iptr->val.a == NULL) {
3508                                         codegen_addpatchref(cd, cd->mcodeptr,
3509                                                                                 PATCHER_builtin_arraycheckcast,
3510                                                                                 (constant_classref *) iptr->target, 0);
3511
3512                                         if (opt_showdisassemble) {
3513                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3514                                         }
3515                                 }
3516
3517                                 M_MOV_IMM(iptr->val.a, rd->argintregs[1]);
3518                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
3519                                 M_CALL(REG_ITMP1);
3520                                 M_TEST(REG_RESULT);
3521                                 M_BEQ(0);
3522                                 codegen_add_classcastexception_ref(cd, cd->mcodeptr);
3523
3524                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3525                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
3526                         }
3527                         M_INTMOVE(s1, d);
3528                         emit_store(jd, iptr, iptr->dst, d);
3529                         break;
3530
3531                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3532
3533                                       /* op1:   0 == array, 1 == class                */
3534                                       /* val.a: (classinfo *) superclass              */
3535
3536                         /*  superclass is an interface:
3537                          *
3538                          *  return (sub != NULL) &&
3539                          *         (sub->vftbl->interfacetablelength > super->index) &&
3540                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3541                          *
3542                          *  superclass is a class:
3543                          *
3544                          *  return ((sub != NULL) && (0
3545                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3546                          *          super->vftbl->diffvall));
3547                          */
3548
3549                         {
3550                         classinfo *super;
3551                         vftbl_t   *supervftbl;
3552                         s4         superindex;
3553
3554                         super = (classinfo *) iptr->val.a;
3555
3556                         if (!super) {
3557                                 superindex = 0;
3558                                 supervftbl = NULL;
3559
3560                         } else {
3561                                 superindex = super->index;
3562                                 supervftbl = super->vftbl;
3563                         }
3564
3565 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3566             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3567 #endif
3568
3569                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3570                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
3571                         if (s1 == d) {
3572                                 M_INTMOVE(s1, REG_ITMP1);
3573                                 s1 = REG_ITMP1;
3574                         }
3575
3576                         /* calculate interface instanceof code size */
3577
3578                         s2 = 3; /* mov_membase_reg */
3579                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3580                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3581                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3582                                 3 /* test */ + 4 /* setcc */;
3583
3584                         if (!super)
3585                                 s2 += (opt_showdisassemble ? 5 : 0);
3586
3587                         /* calculate class instanceof code size */
3588
3589                         s3 = 3; /* mov_membase_reg */
3590                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3591                         s3 += 10; /* mov_imm_reg */
3592                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3593                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3594                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3595                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3596                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3597                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3598                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3599
3600                         if (!super)
3601                                 s3 += (opt_showdisassemble ? 5 : 0);
3602
3603                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3604
3605                         /* if class is not resolved, check which code to call */
3606
3607                         if (!super) {
3608                                 x86_64_test_reg_reg(cd, s1, s1);
3609                                 x86_64_jcc(cd, X86_64_CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3610                                                                                          7 + 6 + s2 + 5 + s3));
3611
3612                                 codegen_addpatchref(cd, cd->mcodeptr,
3613                                                                         PATCHER_checkcast_instanceof_flags,
3614                                                                         (constant_classref *) iptr->target, 0);
3615
3616                                 if (opt_showdisassemble) {
3617                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3618                                 }
3619
3620                                 x86_64_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3621                                 x86_64_alul_imm_reg(cd, X86_64_AND, ACC_INTERFACE, REG_ITMP3);
3622                                 x86_64_jcc(cd, X86_64_CC_Z, s2 + 5);
3623                         }
3624
3625                         /* interface instanceof code */
3626
3627                         if (!super || (super->flags & ACC_INTERFACE)) {
3628                                 if (super) {
3629                                         x86_64_test_reg_reg(cd, s1, s1);
3630                                         x86_64_jcc(cd, X86_64_CC_Z, s2);
3631                                 }
3632
3633                                 x86_64_mov_membase_reg(cd, s1,
3634                                                                            OFFSET(java_objectheader, vftbl),
3635                                                                            REG_ITMP1);
3636                                 if (!super) {
3637                                         codegen_addpatchref(cd, cd->mcodeptr,
3638                                                                                 PATCHER_checkcast_instanceof_interface,
3639                                                                                 (constant_classref *) iptr->target, 0);
3640
3641                                         if (opt_showdisassemble) {
3642                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3643                                         }
3644                                 }
3645
3646                                 x86_64_movl_membase32_reg(cd, REG_ITMP1,
3647                                                                                   OFFSET(vftbl_t, interfacetablelength),
3648                                                                                   REG_ITMP3);
3649                                 x86_64_alu_imm32_reg(cd, X86_64_SUB, superindex, REG_ITMP3);
3650                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3651
3652                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3653
3654                                 x86_64_jcc(cd, X86_64_CC_LE, a);
3655                                 x86_64_mov_membase32_reg(cd, REG_ITMP1,
3656                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3657                                                                                  superindex * sizeof(methodptr*),
3658                                                                                  REG_ITMP1);
3659                                 x86_64_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3660                                 x86_64_setcc_reg(cd, X86_64_CC_NE, d);
3661
3662                                 if (!super)
3663                                         x86_64_jmp_imm(cd, s3);
3664                         }
3665
3666                         /* class instanceof code */
3667
3668                         if (!super || !(super->flags & ACC_INTERFACE)) {
3669                                 if (super) {
3670                                         x86_64_test_reg_reg(cd, s1, s1);
3671                                         x86_64_jcc(cd, X86_64_CC_E, s3);
3672                                 }
3673
3674                                 x86_64_mov_membase_reg(cd, s1,
3675                                                                            OFFSET(java_objectheader, vftbl),
3676                                                                            REG_ITMP1);
3677
3678                                 if (!super) {
3679                                         codegen_addpatchref(cd, cd->mcodeptr,
3680                                                                                 PATCHER_instanceof_class,
3681                                                                                 (constant_classref *) iptr->target, 0);
3682
3683                                         if (opt_showdisassemble) {
3684                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3685                                         }
3686                                 }
3687
3688                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3689 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3690                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3691 #endif
3692                                 x86_64_movl_membase_reg(cd, REG_ITMP1,
3693                                                                                 OFFSET(vftbl_t, baseval),
3694                                                                                 REG_ITMP1);
3695                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3696                                                                                 OFFSET(vftbl_t, diffval),
3697                                                                                 REG_ITMP3);
3698                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3699                                                                                 OFFSET(vftbl_t, baseval),
3700                                                                                 REG_ITMP2);
3701 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3702                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3703 #endif
3704                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
3705                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d); /* may be REG_ITMP2 */
3706                                 x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP3, REG_ITMP1);
3707                                 x86_64_setcc_reg(cd, X86_64_CC_BE, d);
3708                         }
3709                         emit_store(jd, iptr, iptr->dst, d);
3710                         }
3711                         break;
3712
3713                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3714                                          /* op1 = dimension, val.a = class            */
3715
3716                         /* check for negative sizes and copy sizes to stack if necessary  */
3717
3718                         MCODECHECK((10 * 4 * iptr->op1) + 5 + 10 * 8);
3719
3720                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3721                                 /* copy SAVEDVAR sizes to stack */
3722
3723                                 if (src->varkind != ARGVAR) {
3724                                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP1);
3725                                         M_LST(s2, REG_SP, s1 * 8);
3726                                 }
3727                         }
3728
3729                         /* is a patcher function set? */
3730
3731                         if (iptr->val.a == NULL) {
3732                                 codegen_addpatchref(cd, cd->mcodeptr,
3733                                                                         PATCHER_builtin_multianewarray,
3734                                                                         (constant_classref *) iptr->target, 0);
3735
3736                                 if (opt_showdisassemble) {
3737                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3738                                 }
3739
3740                                 a = 0;
3741
3742                         } else {
3743                                 a = (ptrint) iptr->val.a;
3744                         }
3745
3746                         /* a0 = dimension count */
3747
3748                         M_MOV_IMM(iptr->op1, rd->argintregs[0]);
3749
3750                         /* a1 = arrayvftbl */
3751
3752                         M_MOV_IMM(iptr->val.a, rd->argintregs[1]);
3753
3754                         /* a2 = pointer to dimensions = stack pointer */
3755
3756                         M_MOV(REG_SP, rd->argintregs[2]);
3757
3758                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3759                         M_CALL(REG_ITMP1);
3760
3761                         /* check for exception before result assignment */
3762
3763                         M_TEST(REG_RESULT);
3764                         M_BEQ(0);
3765                         codegen_add_fillinstacktrace_ref(cd, cd->mcodeptr);
3766
3767                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_RESULT);
3768                         M_INTMOVE(REG_RESULT, s1);
3769                         emit_store(jd, iptr, iptr->dst, s1);
3770                         break;
3771
3772                 default:
3773                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3774                         return false;
3775         } /* switch */
3776
3777         } /* for instruction */
3778
3779         /* copy values to interface registers */
3780
3781         src = bptr->outstack;
3782         len = bptr->outdepth;
3783         MCODECHECK(512);
3784 #if defined(ENABLE_LSRA)
3785         if (!opt_lsra)
3786 #endif
3787         while (src) {
3788                 len--;
3789                 if ((src->varkind != STACKVAR)) {
3790                         s2 = src->type;
3791                         if (IS_FLT_DBL_TYPE(s2)) {
3792                                 s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
3793                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3794                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3795
3796                                 } else {
3797                                         x86_64_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3798                                 }
3799
3800                         } else {
3801                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3802                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3803                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3804
3805                                 } else {
3806                                         x86_64_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3807                                 }
3808                         }
3809                 }
3810                 src = src->prev;
3811         }
3812
3813         /* At the end of a basic block we may have to append some nops,
3814            because the patcher stub calling code might be longer than the
3815            actual instruction. So codepatching does not change the
3816            following block unintentionally. */
3817
3818         if (cd->mcodeptr < cd->lastmcodeptr) {
3819                 while (cd->mcodeptr < cd->lastmcodeptr) {
3820                         M_NOP;
3821                 }
3822         }
3823
3824         } /* if (bptr -> flags >= BBREACHED) */
3825         } /* for basic block */
3826
3827         dseg_createlinenumbertable(cd);
3828
3829
3830         /* generate exception and patcher stubs */
3831
3832         {
3833                 exceptionref *eref;
3834                 patchref     *pref;
3835                 ptrint        mcode;
3836                 u1           *savedmcodeptr;
3837                 u1           *tmpmcodeptr;
3838
3839                 savedmcodeptr = NULL;
3840
3841                 /* generate exception stubs */
3842
3843                 for (eref = cd->exceptionrefs; eref != NULL; eref = eref->next) {
3844                         gen_resolvebranch(cd->mcodebase + eref->branchpos,
3845                                                           eref->branchpos,
3846                                                           cd->mcodeptr - cd->mcodebase);
3847
3848                         MCODECHECK(512);
3849
3850                         /* Check if the exception is an
3851                            ArrayIndexOutOfBoundsException.  If so, move index register
3852                            into a4. */
3853
3854                         if (eref->reg != -1)
3855                                 M_MOV(eref->reg, rd->argintregs[4]);
3856
3857                         /* calcuate exception address */
3858
3859                         M_MOV_IMM(0, rd->argintregs[3]);
3860                         dseg_adddata(cd, cd->mcodeptr);
3861                         M_AADD_IMM32(eref->branchpos - 6, rd->argintregs[3]);
3862
3863                         /* move function to call into REG_ITMP3 */
3864
3865                         M_MOV_IMM(eref->function, REG_ITMP3);
3866
3867                         if (savedmcodeptr != NULL) {
3868                                 M_JMP_IMM(savedmcodeptr - cd->mcodeptr - 5);
3869
3870                         } else {
3871                                 savedmcodeptr = cd->mcodeptr;
3872
3873                                 x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3874                                 M_MOV(REG_SP, rd->argintregs[1]);
3875                                 M_ALD(rd->argintregs[2], REG_SP, stackframesize * 8);
3876
3877                                 M_ASUB_IMM(2 * 8, REG_SP);
3878                                 M_AST(rd->argintregs[3], REG_SP, 0 * 8);         /* store XPC */
3879
3880                                 M_CALL(REG_ITMP3);
3881
3882                                 M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3883                                 M_AADD_IMM(2 * 8, REG_SP);
3884
3885                                 M_MOV_IMM(asm_handle_exception, REG_ITMP3);
3886                                 M_JMP(REG_ITMP3);
3887                         }
3888                 }
3889
3890
3891                 /* generate code patching stub call code */
3892
3893                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
3894                         /* check size of code segment */
3895
3896                         MCODECHECK(512);
3897
3898                         /* Get machine code which is patched back in later. A
3899                            `call rel32' is 5 bytes long (but read 8 bytes). */
3900
3901                         savedmcodeptr = cd->mcodebase + pref->branchpos;
3902                         mcode = *((ptrint *) savedmcodeptr);
3903
3904                         /* patch in `call rel32' to call the following code */
3905
3906                         tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr          */
3907                         cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position */
3908
3909                         M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
3910
3911                         cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr   */
3912
3913                         /* move pointer to java_objectheader onto stack */
3914
3915 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3916                         /* create a virtual java_objectheader */
3917
3918                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
3919                         a = dseg_addaddress(cd, NULL);                      /* vftbl      */
3920
3921                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP3);
3922                         M_PUSH(REG_ITMP3);
3923 #else
3924                         M_PUSH_IMM(0);
3925 #endif
3926
3927                         /* move machine code bytes and classinfo pointer into registers */
3928
3929                         M_MOV_IMM(mcode, REG_ITMP3);
3930                         M_PUSH(REG_ITMP3);
3931                         M_MOV_IMM(pref->ref, REG_ITMP3);
3932                         M_PUSH(REG_ITMP3);
3933                         M_MOV_IMM(pref->disp, REG_ITMP3);
3934                         M_PUSH(REG_ITMP3);
3935
3936                         M_MOV_IMM(pref->patcher, REG_ITMP3);
3937                         M_PUSH(REG_ITMP3);
3938
3939                         M_MOV_IMM(asm_wrapper_patcher, REG_ITMP3);
3940                         M_JMP(REG_ITMP3);
3941                 }
3942         }
3943
3944         /* generate replacement-out stubs */
3945
3946         {
3947                 int i;
3948
3949                 replacementpoint = jd->code->rplpoints;
3950
3951                 for (i = 0; i < jd->code->rplpointcount; ++i, ++replacementpoint) {
3952                         /* check code segment size */
3953
3954                         MCODECHECK(512);
3955
3956                         /* note start of stub code */
3957
3958                         replacementpoint->outcode = (u1*) (ptrint)(cd->mcodeptr - cd->mcodebase);
3959
3960                         /* make machine code for patching */
3961
3962                         disp = (ptrint)(replacementpoint->outcode - replacementpoint->pc) - 5;
3963                         replacementpoint->mcode = 0xe9 | ((u8)disp << 8);
3964
3965                         /* push address of `rplpoint` struct */
3966
3967                         M_MOV_IMM(replacementpoint, REG_ITMP3);
3968                         M_PUSH(REG_ITMP3);
3969
3970                         /* jump to replacement function */
3971
3972                         M_MOV_IMM(asm_replacement_out, REG_ITMP3);
3973                         M_JMP(REG_ITMP3);
3974                 }
3975         }
3976
3977         codegen_finish(jd, (s4) (cd->mcodeptr - cd->mcodebase));
3978
3979         /* everything's ok */
3980
3981         return true;
3982 }
3983
3984
3985 /* createcompilerstub **********************************************************
3986
3987    Creates a stub routine which calls the compiler.
3988
3989 *******************************************************************************/
3990
3991 #define COMPILERSTUB_DATASIZE    2 * SIZEOF_VOID_P
3992 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
3993
3994 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
3995
3996
3997 u1 *createcompilerstub(methodinfo *m)
3998 {
3999         u1          *s;                     /* memory to hold the stub            */
4000         ptrint      *d;
4001         codegendata *cd;
4002         s4           dumpsize;
4003
4004         s = CNEW(u1, COMPILERSTUB_SIZE);
4005
4006         /* set data pointer and code pointer */
4007
4008         d = (ptrint *) s;
4009         s = s + COMPILERSTUB_DATASIZE;
4010
4011         /* mark start of dump memory area */
4012
4013         dumpsize = dump_size();
4014
4015         cd = DNEW(codegendata);
4016         cd->mcodeptr = s;
4017
4018         /* Store the methodinfo* in the same place as in the methodheader
4019            for compiled methods. */
4020
4021         d[0] = (ptrint) asm_call_jit_compiler;
4022         d[1] = (ptrint) m;
4023
4024         /* code for the stub */
4025
4026         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 1 * SIZEOF_VOID_P));       /* methodinfo  */
4027         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 2 * SIZEOF_VOID_P));  /* compiler pointer */
4028         M_JMP(REG_ITMP3);
4029
4030 #if defined(ENABLE_STATISTICS)
4031         if (opt_stat)
4032                 count_cstub_len += COMPILERSTUB_SIZE;
4033 #endif
4034
4035         /* release dump area */
4036
4037         dump_release(dumpsize);
4038
4039         return s;
4040 }
4041
4042
4043 /* createnativestub ************************************************************
4044
4045    Creates a stub routine which calls a native method.
4046
4047 *******************************************************************************/
4048
4049 u1 *createnativestub(functionptr f, jitdata *jd, methoddesc *nmd)
4050 {
4051         methodinfo   *m;
4052         codegendata  *cd;
4053         registerdata *rd;
4054         methoddesc   *md;
4055         s4            stackframesize;       /* size of stackframe if needed       */
4056         s4            nativeparams;
4057         s4            i, j;                 /* count variables                    */
4058         s4            t;
4059         s4            s1, s2;
4060
4061         /* get required compiler data */
4062
4063         m  = jd->m;
4064         cd = jd->cd;
4065         rd = jd->rd;
4066
4067         /* initialize variables */
4068
4069         md = m->parseddesc;
4070         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
4071
4072         /* calculate stack frame size */
4073
4074         stackframesize =
4075                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
4076                 sizeof(localref_table) / SIZEOF_VOID_P +
4077                 INT_ARG_CNT + FLT_ARG_CNT + 1 +         /* + 1 for function address   */
4078                 nmd->memuse;
4079
4080         if (!(stackframesize & 0x1))                /* keep stack 16-byte aligned */
4081                 stackframesize++;
4082
4083         /* create method header */
4084
4085         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
4086         (void) dseg_adds4(cd, stackframesize * 8);              /* FrameSize      */
4087         (void) dseg_adds4(cd, 0);                               /* IsSync         */
4088         (void) dseg_adds4(cd, 0);                               /* IsLeaf         */
4089         (void) dseg_adds4(cd, 0);                               /* IntSave        */
4090         (void) dseg_adds4(cd, 0);                               /* FltSave        */
4091         (void) dseg_addlinenumbertablesize(cd);
4092         (void) dseg_adds4(cd, 0);                               /* ExTableSize    */
4093
4094         /* generate native method profiling code */
4095
4096         if (opt_prof) {
4097                 /* count frequency */
4098
4099                 M_MOV_IMM(m, REG_ITMP2);
4100                 M_IINC_MEMBASE(REG_ITMP2, OFFSET(methodinfo, frequency));
4101         }
4102
4103         /* generate stub code */
4104
4105         M_ASUB_IMM(stackframesize * 8, REG_SP);
4106
4107 #if !defined(NDEBUG)
4108         /* generate call trace */
4109
4110         if (opt_verbosecall) {
4111                 /* save integer and float argument registers */
4112
4113                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4114                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4115                                 M_LST(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4116
4117                 for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4118                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4119                                 M_DST(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4120
4121                 /* show integer hex code for float arguments */
4122
4123                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++) {
4124                         /* if the paramtype is a float, we have to right shift all
4125                            following integer registers */
4126
4127                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type)) {
4128                                 for (s1 = INT_ARG_CNT - 2; s1 >= i; s1--)
4129                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
4130
4131                                 x86_64_movd_freg_reg(cd, rd->argfltregs[j], rd->argintregs[i]);
4132                                 j++;
4133                         }
4134                 }
4135
4136                 M_MOV_IMM(m, REG_ITMP1);
4137                 M_AST(REG_ITMP1, REG_SP, 0 * 8);
4138                 M_MOV_IMM(builtin_trace_args, REG_ITMP1);
4139                 M_CALL(REG_ITMP1);
4140
4141                 /* restore integer and float argument registers */
4142
4143                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4144                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4145                                 M_LLD(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4146
4147                 for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4148                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4149                                 M_DLD(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4150         }
4151 #endif /* !defined(NDEBUG) */
4152
4153         /* get function address (this must happen before the stackframeinfo) */
4154
4155 #if !defined(WITH_STATIC_CLASSPATH)
4156         if (f == NULL) {
4157                 codegen_addpatchref(cd, cd->mcodeptr, PATCHER_resolve_native, m, 0);
4158
4159                 if (opt_showdisassemble) {
4160                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
4161                 }
4162         }
4163 #endif
4164
4165         M_MOV_IMM(f, REG_ITMP3);
4166
4167
4168         /* save integer and float argument registers */
4169
4170         for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4171                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4172                         M_LST(rd->argintregs[j++], REG_SP, i * 8);
4173
4174         for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4175                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4176                         M_DST(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4177
4178         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4179
4180         /* create dynamic stack info */
4181
4182         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4183         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[1]);
4184         M_ALEA(REG_SP, stackframesize * 8 + SIZEOF_VOID_P, rd->argintregs[2]);
4185         M_ALD(rd->argintregs[3], REG_SP, stackframesize * 8);
4186         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
4187         M_CALL(REG_ITMP1);
4188
4189         /* restore integer and float argument registers */
4190
4191         for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4192                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4193                         M_LLD(rd->argintregs[j++], REG_SP, i * 8);
4194
4195         for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4196                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4197                         M_DLD(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4198
4199         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4200
4201
4202         /* copy or spill arguments to new locations */
4203
4204         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
4205                 t = md->paramtypes[i].type;
4206
4207                 if (IS_INT_LNG_TYPE(t)) {
4208                         if (!md->params[i].inmemory) {
4209                                 s1 = rd->argintregs[md->params[i].regoff];
4210
4211                                 if (!nmd->params[j].inmemory) {
4212                                         s2 = rd->argintregs[nmd->params[j].regoff];
4213                                         M_INTMOVE(s1, s2);
4214
4215                                 } else {
4216                                         s2 = nmd->params[j].regoff;
4217                                         M_LST(s1, REG_SP, s2 * 8);
4218                                 }
4219
4220                         } else {
4221                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4222                                 s2 = nmd->params[j].regoff;
4223                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
4224                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
4225                         }
4226
4227                 } else {
4228                         /* We only copy spilled float arguments, as the float argument    */
4229                         /* registers keep unchanged.                                      */
4230
4231                         if (md->params[i].inmemory) {
4232                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4233                                 s2 = nmd->params[j].regoff;
4234                                 M_DLD(REG_FTMP1, REG_SP, s1 * 8);
4235                                 M_DST(REG_FTMP1, REG_SP, s2 * 8);
4236                         }
4237                 }
4238         }
4239
4240         /* put class into second argument register */
4241
4242         if (m->flags & ACC_STATIC)
4243                 M_MOV_IMM(m->class, rd->argintregs[1]);
4244
4245         /* put env into first argument register */
4246
4247         M_MOV_IMM(_Jv_env, rd->argintregs[0]);
4248
4249         /* do the native function call */
4250
4251         M_CALL(REG_ITMP3);
4252
4253         /* save return value */
4254
4255         if (md->returntype.type != TYPE_VOID) {
4256                 if (IS_INT_LNG_TYPE(md->returntype.type))
4257                         M_LST(REG_RESULT, REG_SP, 0 * 8);
4258                 else
4259                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
4260         }
4261
4262         /* remove native stackframe info */
4263
4264         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4265         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
4266         M_CALL(REG_ITMP1);
4267
4268 #if !defined(NDEBUG)
4269         /* generate call trace */
4270
4271         if (opt_verbosecall) {
4272                 /* just restore the value we need, don't care about the other */
4273
4274                 if (md->returntype.type != TYPE_VOID) {
4275                         if (IS_INT_LNG_TYPE(md->returntype.type))
4276                                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
4277                         else
4278                                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4279                 }
4280
4281                 M_MOV_IMM(m, rd->argintregs[0]);
4282                 M_MOV(REG_RESULT, rd->argintregs[1]);
4283                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4284                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4285
4286                 M_MOV_IMM(builtin_displaymethodstop, REG_ITMP1);
4287                 M_CALL(REG_ITMP1);
4288         }
4289 #endif /* !defined(NDEBUG) */
4290
4291         /* check for exception */
4292
4293 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4294         M_MOV_IMM(builtin_get_exceptionptrptr, REG_ITMP3);
4295         M_CALL(REG_ITMP3);
4296 #else
4297         M_MOV_IMM(&_no_threads_exceptionptr, REG_RESULT);
4298 #endif
4299         M_ALD(REG_ITMP2, REG_RESULT, 0);
4300
4301         /* restore return value */
4302
4303         if (md->returntype.type != TYPE_VOID) {
4304                 if (IS_INT_LNG_TYPE(md->returntype.type))
4305                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
4306                 else
4307                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4308         }
4309
4310         /* test for exception */
4311
4312         M_TEST(REG_ITMP2);
4313         M_BNE(7 + 1);
4314
4315         /* remove stackframe */
4316
4317         M_AADD_IMM(stackframesize * 8, REG_SP);
4318         M_RET;
4319
4320
4321         /* handle exception */
4322
4323 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4324         M_LST(REG_ITMP2, REG_SP, 0 * 8);
4325         M_MOV_IMM(builtin_get_exceptionptrptr, REG_ITMP3);
4326         M_CALL(REG_ITMP3);
4327         M_AST_IMM32(0, REG_RESULT, 0);                 /* clear exception pointer */
4328         M_LLD(REG_ITMP1_XPTR, REG_SP, 0 * 8);
4329 #else
4330         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
4331         M_MOV_IMM(&_no_threads_exceptionptr, REG_ITMP3);
4332         M_AST_IMM32(0, REG_ITMP3, 0);                  /* clear exception pointer */
4333 #endif
4334
4335         /* remove stackframe */
4336
4337         M_AADD_IMM(stackframesize * 8, REG_SP);
4338
4339         M_LLD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
4340         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
4341
4342         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
4343         M_JMP(REG_ITMP3);
4344
4345
4346         /* process patcher calls **************************************************/
4347
4348         {
4349                 patchref *pref;
4350                 ptrint    mcode;
4351                 u1       *savedmcodeptr;
4352                 u1       *tmpmcodeptr;
4353 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4354                 s4        disp;
4355 #endif
4356
4357                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4358                         /* Get machine code which is patched back in later. A
4359                            `call rel32' is 5 bytes long (but read 8 bytes). */
4360
4361                         savedmcodeptr = cd->mcodebase + pref->branchpos;
4362                         mcode = *((ptrint *) savedmcodeptr);
4363
4364                         /* patch in `call rel32' to call the following code               */
4365
4366                         tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr          */
4367                         cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position */
4368
4369                         M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
4370
4371                         cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr   */
4372
4373                         /* move pointer to java_objectheader onto stack */
4374
4375 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4376                         /* create a virtual java_objectheader */
4377
4378                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4379                         disp = dseg_addaddress(cd, NULL);                   /* vftbl      */
4380
4381                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP3);
4382                         M_PUSH(REG_ITMP3);
4383 #else
4384                         M_PUSH_IMM(0);
4385 #endif
4386
4387                         /* move machine code bytes and classinfo pointer into registers */
4388
4389                         M_MOV_IMM(mcode, REG_ITMP3);
4390                         M_PUSH(REG_ITMP3);
4391                         M_MOV_IMM(pref->ref, REG_ITMP3);
4392                         M_PUSH(REG_ITMP3);
4393                         M_MOV_IMM(pref->disp, REG_ITMP3);
4394                         M_PUSH(REG_ITMP3);
4395
4396                         M_MOV_IMM(pref->patcher, REG_ITMP3);
4397                         M_PUSH(REG_ITMP3);
4398
4399                         M_MOV_IMM(asm_wrapper_patcher, REG_ITMP3);
4400                         M_JMP(REG_ITMP3);
4401                 }
4402         }
4403
4404         codegen_finish(jd, (s4) (cd->mcodeptr - cd->mcodebase));
4405
4406         return jd->code->entrypoint;
4407 }
4408
4409
4410 /*
4411  * These are local overrides for various environment variables in Emacs.
4412  * Please do not remove this and leave it at the end of the file, where
4413  * Emacs will automagically detect them.
4414  * ---------------------------------------------------------------------
4415  * Local variables:
4416  * mode: c
4417  * indent-tabs-mode: t
4418  * c-basic-offset: 4
4419  * tab-width: 4
4420  * End:
4421  * vim:noexpandtab:sw=4:ts=4:
4422  */