src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25 */
  26
  27
  28 #include "config.h"
  29
  30 #include <assert.h>
  31 #include <stdio.h>
  32
  33 #include "vm/types.h"
  34
  35 #include "md-abi.h"
  36
  37 #include "vm/jit/x86_64/arch.h"
  38 #include "vm/jit/x86_64/codegen.h"
  39 #include "vm/jit/x86_64/emit.h"
  40
  41 #include "mm/memory.h"
  42
  43 #include "native/jni.h"
  44 #include "native/localref.h"
  45 #include "native/native.h"
  46
  47 #include "threads/lock-common.h"
  48
  49 #include "vm/builtin.h"
  50 #include "vm/exceptions.h"
  51 #include "vm/global.h"
  52 #include "vm/stringlocal.h"
  53 #include "vm/vm.h"
  54
  55 #include "vm/jit/abi.h"
  56 #include "vm/jit/asmpart.h"
  57 #include "vm/jit/codegen-common.h"
  58 #include "vm/jit/dseg.h"
  59 #include "vm/jit/emit-common.h"
  60 #include "vm/jit/jit.h"
  61 #include "vm/jit/methodheader.h"
  62 #include "vm/jit/parse.h"
  63 #include "vm/jit/patcher.h"
  64 #include "vm/jit/reg.h"
  65 #include "vm/jit/replace.h"
  66 #include "vm/jit/stacktrace.h"
  67
  68 #if defined(ENABLE_LSRA)
  69 # include "vm/jit/allocator/lsra.h"
  70 #endif
  71
  72 #include "vmcore/loader.h"
  73 #include "vmcore/options.h"
  74 #include "vmcore/statistics.h"
  75
  76
  77 /* codegen_emit ****************************************************************
  78
  79    Generates machine code.
  80
  81 *******************************************************************************/
  82
  83 bool codegen_emit(jitdata *jd)
  84 {
  85         methodinfo         *m;
  86         codeinfo           *code;
  87         codegendata        *cd;
  88         registerdata       *rd;
  89         s4                  len, s1, s2, s3, d, disp;
  90         u2                  currentline;
  91         ptrint              a;
  92         varinfo            *var, *dst;
  93         basicblock         *bptr;
  94         instruction        *iptr;
  95         exception_entry    *ex;
  96         constant_classref  *cr;
  97         unresolved_class   *uc;
  98         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
  99         unresolved_method  *um;
 100         builtintable_entry *bte;
 101         methoddesc         *md;
 102         fieldinfo          *fi;
 103         unresolved_field   *uf;
 104         s4                  fieldtype;
 105         s4                 varindex;
 106
 107         /* get required compiler data */
 108
 109         m    = jd->m;
 110         code = jd->code;
 111         cd   = jd->cd;
 112         rd   = jd->rd;
 113
 114         /* prevent compiler warnings */
 115
 116         d   = 0;
 117         lm  = NULL;
 118         um  = NULL;
 119         bte = NULL;
 120
 121         {
 122         s4 i, p, t, l;
 123         s4 savedregs_num;
 124
 125         savedregs_num = 0;
 126
 127         /* space to save used callee saved registers */
 128
 129         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 130         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 131
 132         cd->stackframesize = rd->memuse + savedregs_num;
 133
 134 #if defined(ENABLE_THREADS)
 135         /* space to save argument of monitor_enter */
 136
 137         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 138                 cd->stackframesize++;
 139 #endif
 140
 141         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 142            native code e.g. libc or jni (alignment problems with
 143            movaps). */
 144
 145         if (!jd->isleafmethod || opt_verbosecall)
 146                 cd->stackframesize |= 0x1;
 147
 148         /* create method header */
 149
 150         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 151         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 152
 153 #if defined(ENABLE_THREADS)
 154         /* IsSync contains the offset relative to the stack pointer for the
 155            argument of monitor_exit used in the exception handler. Since the
 156            offset could be zero and give a wrong meaning of the flag it is
 157            offset by one.
 158         */
 159
 160         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 161                 (void) dseg_add_unique_s4(cd, (rd->memuse + 1) * 8); /* IsSync        */
 162         else
 163 #endif
 164                 (void) dseg_add_unique_s4(cd, 0);                  /* IsSync          */
 165
 166         (void) dseg_add_unique_s4(cd, jd->isleafmethod);               /* IsLeaf  */
 167         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 168         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 169
 170         (void) dseg_addlinenumbertablesize(cd);
 171
 172         (void) dseg_add_unique_s4(cd, jd->exceptiontablelength); /* ExTableSize   */
 173
 174         /* create exception table */
 175
 176         for (ex = jd->exceptiontable; ex != NULL; ex = ex->down) {
 177                 dseg_add_target(cd, ex->start);
 178                 dseg_add_target(cd, ex->end);
 179                 dseg_add_target(cd, ex->handler);
 180                 (void) dseg_add_unique_address(cd, ex->catchtype.any);
 181         }
 182
 183 #if defined(ENABLE_PROFILING)
 184         /* generate method profiling code */
 185
 186         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 187                 /* count frequency */
 188
 189                 M_MOV_IMM(code, REG_ITMP3);
 190                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 191
 192                 PROFILE_CYCLE_START;
 193         }
 194 #endif
 195
 196         /* create stack frame (if necessary) */
 197
 198         if (cd->stackframesize)
 199                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 200
 201         /* save used callee saved registers */
 202
 203         p = cd->stackframesize;
 204         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 205                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 206         }
 207         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 208                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 209         }
 210
 211         /* take arguments out of register or stack frame */
 212
 213         md = m->parseddesc;
 214
 215         for (p = 0, l = 0; p < md->paramcount; p++) {
 216                 t = md->paramtypes[p].type;
 217
 218                 varindex = jd->local_map[l * 5 + t];
 219
 220                 l++;
 221                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 222                         l++;
 223
 224                 if (varindex == UNUSED)
 225                         continue;
 226
 227                 var = VAR(varindex);
 228
 229                 s1 = md->params[p].regoff;
 230
 231                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 232                         if (!md->params[p].inmemory) {           /* register arguments    */
 233                                 if (!IS_INMEMORY(var->flags))
 234                                         M_INTMOVE(s1, var->vv.regoff);
 235                                 else
 236                                     M_LST(s1, REG_SP, var->vv.regoff);
 237                         }
 238                         else {                                 /* stack arguments       */
 239                                 if (!IS_INMEMORY(var->flags))
 240                                         /* + 8 for return address */
 241                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 242                                 else
 243                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 244                         }
 245                 }
 246                 else {                                     /* floating args         */
 247                         if (!md->params[p].inmemory) {           /* register arguments    */
 248                                 if (!IS_INMEMORY(var->flags))
 249                                         M_FLTMOVE(s1, var->vv.regoff);
 250                                 else
 251                                         M_DST(s1, REG_SP, var->vv.regoff);
 252                         }
 253                         else {                                 /* stack arguments       */
 254                                 if (!IS_INMEMORY(var->flags))
 255                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 256                                 else
 257                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 258                         }
 259                 }
 260         }
 261
 262         /* save monitorenter argument */
 263
 264 #if defined(ENABLE_THREADS)
 265         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 266                 /* stack offset for monitor argument */
 267
 268                 s1 = rd->memuse;
 269
 270                 if (opt_verbosecall) {
 271                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 272
 273                         for (p = 0; p < INT_ARG_CNT; p++)
 274                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 275
 276                         for (p = 0; p < FLT_ARG_CNT; p++)
 277                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 278
 279                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 280                 }
 281
 282                 /* decide which monitor enter function to call */
 283
 284                 if (m->flags & ACC_STATIC) {
 285                         M_MOV_IMM(&m->class->object.header, REG_A0);
 286                 }
 287                 else {
 288                         M_TEST(REG_A0);
 289                         M_BNE(8);
 290                         M_ALD_MEM(REG_A0, EXCEPTION_HARDWARE_NULLPOINTER);
 291                 }
 292
 293                 M_AST(REG_A0, REG_SP, s1 * 8);
 294                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 295                 M_CALL(REG_ITMP1);
 296
 297                 if (opt_verbosecall) {
 298                         for (p = 0; p < INT_ARG_CNT; p++)
 299                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 300
 301                         for (p = 0; p < FLT_ARG_CNT; p++)
 302                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 303
 304                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 305                 }
 306         }
 307 #endif
 308
 309 #if !defined(NDEBUG)
 310         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 311                 emit_verbosecall_enter(jd);
 312 #endif /* !defined(NDEBUG) */
 313
 314         }
 315
 316         /* end of header generation */
 317
 318         /* create replacement points */
 319
 320         REPLACEMENT_POINTS_INIT(cd, jd);
 321
 322         /* walk through all basic blocks */
 323
 324         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 325
 326                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 327
 328                 if (bptr->flags >= BBREACHED) {
 329
 330                 /* branch resolving */
 331
 332                 codegen_resolve_branchrefs(cd, bptr);
 333
 334                 /* handle replacement points */
 335
 336                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 337
 338                 /* copy interface registers to their destination */
 339
 340                 len = bptr->indepth;
 341                 MCODECHECK(512);
 342
 343 #if defined(ENABLE_PROFILING)
 344                 /* generate basicblock profiling code */
 345
 346                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 347                         /* count frequency */
 348
 349                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 350                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 351
 352                         /* if this is an exception handler, start profiling again */
 353
 354                         if (bptr->type == BBTYPE_EXH)
 355                                 PROFILE_CYCLE_START;
 356                 }
 357 #endif
 358
 359 #if defined(ENABLE_LSRA)
 360                 if (opt_lsra) {
 361                         while (len) {
 362                                 len--;
 363                                 src = bptr->invars[len];
 364                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 365                                         if (bptr->type == BBTYPE_EXH) {
 366 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 367                                                 if (!IS_INMEMORY(src->flags))
 368                                                         d= src->vv.regoff;
 369                                                 else
 370                                                         d=REG_ITMP1;
 371                                                 M_INTMOVE(REG_ITMP1, d);
 372                                                 emit_store(jd, NULL, src, d);
 373                                         }
 374                                 }
 375                         }
 376
 377                 } else {
 378 #endif
 379
 380                 while (len) {
 381                         len--;
 382                         var = VAR(bptr->invars[len]);
 383                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 384                                 if (bptr->type == BBTYPE_EXH) {
 385                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 386                                         M_INTMOVE(REG_ITMP1, d);
 387                                         emit_store(jd, NULL, var, d);
 388                                 }
 389                         }
 390                         else {
 391                                 assert((var->flags & INOUT));
 392                         }
 393                 }
 394 #if defined(ENABLE_LSRA)
 395                 }
 396 #endif
 397                 /* walk through all instructions */
 398
 399                 len = bptr->icount;
 400                 currentline = 0;
 401
 402                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 403                         if (iptr->line != currentline) {
 404                                 dseg_addlinenumber(cd, iptr->line);
 405                                 currentline = iptr->line;
 406                         }
 407
 408                         MCODECHECK(1024);                         /* 1KB should be enough */
 409
 410                 switch (iptr->opc) {
 411                 case ICMD_NOP:        /* ...  ==> ...                                 */
 412                 case ICMD_POP:        /* ..., value  ==> ...                          */
 413                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 414                         break;
 415
 416                 case ICMD_INLINE_START:
 417
 418                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 419                         break;
 420
 421                 case ICMD_INLINE_BODY:
 422
 423                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 424                         dseg_addlinenumber_inline_start(cd, iptr);
 425                         dseg_addlinenumber(cd, iptr->line);
 426                         break;
 427
 428                 case ICMD_INLINE_END:
 429
 430                         dseg_addlinenumber_inline_end(cd, iptr);
 431                         dseg_addlinenumber(cd, iptr->line);
 432                         break;
 433
 434                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 435
 436                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 437                         emit_nullpointer_check(cd, iptr, s1);
 438                         break;
 439
 440                 /* constant operations ************************************************/
 441
 442                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 443
 444                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 445                         ICONST(d, iptr->sx.val.i);
 446                         emit_store_dst(jd, iptr, d);
 447                         break;
 448
 449                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 450
 451                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 452                         LCONST(d, iptr->sx.val.l);
 453                         emit_store_dst(jd, iptr, d);
 454                         break;
 455
 456                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 457
 458                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 459                         disp = dseg_add_float(cd, iptr->sx.val.f);
 460                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 461                         emit_store_dst(jd, iptr, d);
 462                         break;
 463
 464                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 465
 466                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 467                         disp = dseg_add_double(cd, iptr->sx.val.d);
 468                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 469                         emit_store_dst(jd, iptr, d);
 470                         break;
 471
 472                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 473
 474                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 475
 476                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 477                                 cr = iptr->sx.val.c.ref;
 478
 479 /*                              PROFILE_CYCLE_STOP; */
 480
 481                                 codegen_add_patch_ref(cd, PATCHER_aconst, cr, 0);
 482
 483 /*                              PROFILE_CYCLE_START; */
 484
 485                                 M_MOV_IMM(NULL, d);
 486
 487                         } else {
 488                                 if (iptr->sx.val.anyptr == 0)
 489                                         M_CLR(d);
 490                                 else
 491                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 492                         }
 493                         emit_store_dst(jd, iptr, d);
 494                         break;
 495
 496
 497                 /* load/store/copy/move operations ************************************/
 498
 499                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 500                 case ICMD_ALOAD:      /* s1 = local variable                          */
 501                 case ICMD_LLOAD:
 502                 case ICMD_FLOAD:
 503                 case ICMD_DLOAD:
 504                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 505                 case ICMD_LSTORE:
 506                 case ICMD_FSTORE:
 507                 case ICMD_DSTORE:
 508                 case ICMD_COPY:
 509                 case ICMD_MOVE:
 510
 511                         emit_copy(jd, iptr);
 512                         break;
 513
 514                 case ICMD_ASTORE:
 515                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 516                                 emit_copy(jd, iptr);
 517                         break;
 518
 519                 /* integer operations *************************************************/
 520
 521                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 522
 523                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 524                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 525                         M_INTMOVE(s1, d);
 526                         M_INEG(d);
 527                         emit_store_dst(jd, iptr, d);
 528                         break;
 529
 530                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 531
 532                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 533                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 534                         M_INTMOVE(s1, d);
 535                         M_LNEG(d);
 536                         emit_store_dst(jd, iptr, d);
 537                         break;
 538
 539                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 540
 541                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 542                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 543                         M_ISEXT(s1, d);
 544                         emit_store_dst(jd, iptr, d);
 545                         break;
 546
 547                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 548
 549                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 550                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 551                         M_IMOV(s1, d);
 552                         emit_store_dst(jd, iptr, d);
 553                         break;
 554
 555                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 556
 557                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 558                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 559                         M_BSEXT(s1, d);
 560                         emit_store_dst(jd, iptr, d);
 561                         break;
 562
 563                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 564
 565                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 566                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 567                         M_CZEXT(s1, d);
 568                         emit_store_dst(jd, iptr, d);
 569                         break;
 570
 571                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 572
 573                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 574                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 575                         M_SSEXT(s1, d);
 576                         emit_store_dst(jd, iptr, d);
 577                         break;
 578
 579
 580                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 581
 582                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 583                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 584                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 585                         if (s2 == d)
 586                                 M_IADD(s1, d);
 587                         else {
 588                                 M_INTMOVE(s1, d);
 589                                 M_IADD(s2, d);
 590                         }
 591                         emit_store_dst(jd, iptr, d);
 592                         break;
 593
 594                 case ICMD_IINC:
 595                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 596                                       /* sx.val.i = constant                             */
 597
 598                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 599                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 600
 601                         /* Using inc and dec is not faster than add (tested with
 602                            sieve). */
 603
 604                         M_INTMOVE(s1, d);
 605                         M_IADD_IMM(iptr->sx.val.i, d);
 606                         emit_store_dst(jd, iptr, d);
 607                         break;
 608
 609                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 610
 611                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 612                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 613                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 614                         if (s2 == d)
 615                                 M_LADD(s1, d);
 616                         else {
 617                                 M_INTMOVE(s1, d);
 618                                 M_LADD(s2, d);
 619                         }
 620                         emit_store_dst(jd, iptr, d);
 621                         break;
 622
 623                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 624                                       /* sx.val.l = constant                             */
 625
 626                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 627                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 628                         M_INTMOVE(s1, d);
 629                         if (IS_IMM32(iptr->sx.val.l))
 630                                 M_LADD_IMM(iptr->sx.val.l, d);
 631                         else {
 632                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 633                                 M_LADD(REG_ITMP2, d);
 634                         }
 635                         emit_store_dst(jd, iptr, d);
 636                         break;
 637
 638                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 639
 640                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 641                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 642                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 643                         if (s2 == d) {
 644                                 M_INTMOVE(s1, REG_ITMP1);
 645                                 M_ISUB(s2, REG_ITMP1);
 646                                 M_INTMOVE(REG_ITMP1, d);
 647                         } else {
 648                                 M_INTMOVE(s1, d);
 649                                 M_ISUB(s2, d);
 650                         }
 651                         emit_store_dst(jd, iptr, d);
 652                         break;
 653
 654                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 655                                       /* sx.val.i = constant                             */
 656
 657                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 658                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 659                         M_INTMOVE(s1, d);
 660                         M_ISUB_IMM(iptr->sx.val.i, d);
 661                         emit_store_dst(jd, iptr, d);
 662                         break;
 663
 664                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 665
 666                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 667                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 668                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 669                         if (s2 == d) {
 670                                 M_INTMOVE(s1, REG_ITMP1);
 671                                 M_LSUB(s2, REG_ITMP1);
 672                                 M_INTMOVE(REG_ITMP1, d);
 673                         } else {
 674                                 M_INTMOVE(s1, d);
 675                                 M_LSUB(s2, d);
 676                         }
 677                         emit_store_dst(jd, iptr, d);
 678                         break;
 679
 680                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 681                                       /* sx.val.l = constant                             */
 682
 683                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 684                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 685                         M_INTMOVE(s1, d);
 686                         if (IS_IMM32(iptr->sx.val.l))
 687                                 M_LSUB_IMM(iptr->sx.val.l, d);
 688                         else {
 689                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 690                                 M_LSUB(REG_ITMP2, d);
 691                         }
 692                         emit_store_dst(jd, iptr, d);
 693                         break;
 694
 695                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 696
 697                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 698                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 699                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 700                         if (s2 == d)
 701                                 M_IMUL(s1, d);
 702                         else {
 703                                 M_INTMOVE(s1, d);
 704                                 M_IMUL(s2, d);
 705                         }
 706                         emit_store_dst(jd, iptr, d);
 707                         break;
 708
 709                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 710                                       /* sx.val.i = constant                             */
 711
 712                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 713                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 714                         if (iptr->sx.val.i == 2) {
 715                                 M_INTMOVE(s1, d);
 716                                 M_ISLL_IMM(1, d);
 717                         } else
 718                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 719                         emit_store_dst(jd, iptr, d);
 720                         break;
 721
 722                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 723
 724                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 725                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 726                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 727                         if (s2 == d)
 728                                 M_LMUL(s1, d);
 729                         else {
 730                                 M_INTMOVE(s1, d);
 731                                 M_LMUL(s2, d);
 732                         }
 733                         emit_store_dst(jd, iptr, d);
 734                         break;
 735
 736                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 737                                       /* sx.val.l = constant                             */
 738
 739                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 740                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 741                         if (IS_IMM32(iptr->sx.val.l))
 742                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 743                         else {
 744                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 745                                 M_INTMOVE(s1, d);
 746                                 M_LMUL(REG_ITMP2, d);
 747                         }
 748                         emit_store_dst(jd, iptr, d);
 749                         break;
 750
 751                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 752
 753                         s1 = emit_load_s1(jd, iptr, RAX);
 754                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 755                         d = codegen_reg_of_dst(jd, iptr, RAX);
 756
 757                         M_INTMOVE(s1, RAX);
 758                         M_INTMOVE(s2, REG_ITMP3);
 759                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 760
 761                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 762
 763                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 764                         M_BNE(4 + 6);
 765                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 766                         M_BEQ(1 + 3);                                          /* 6 bytes */
 767
 768                         emit_cltd(cd);                                         /* 1 byte  */
 769                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 770
 771                         M_INTMOVE(RAX, d);
 772                         emit_store_dst(jd, iptr, d);
 773                         dst = VAROP(iptr->dst);
 774                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 775                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 776                         break;
 777
 778                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 779
 780                         s1 = emit_load_s1(jd, iptr, RAX);
 781                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 782                         d = codegen_reg_of_dst(jd, iptr, RDX);
 783
 784                         M_INTMOVE(s1, RAX);
 785                         M_INTMOVE(s2, REG_ITMP3);
 786                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 787
 788                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 789
 790                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 791                         M_BNE(3 + 4 + 6);
 792                         M_CLR(RDX);                                            /* 3 bytes */
 793                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 794                         M_BEQ(1 + 3);                                          /* 6 bytes */
 795
 796                         emit_cltd(cd);                                         /* 1 byte  */
 797                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 798
 799                         M_INTMOVE(RDX, d);
 800                         emit_store_dst(jd, iptr, d);
 801                         dst = VAROP(iptr->dst);
 802                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 803                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 804                         break;
 805
 806                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 807                                       /* sx.val.i = constant                             */
 808
 809                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 810                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 811                         M_INTMOVE(s1, REG_ITMP1);
 812                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 813                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 814                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 815                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 816                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 817                         emit_store_dst(jd, iptr, d);
 818                         break;
 819
 820                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 821                                       /* sx.val.i = constant                             */
 822
 823                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 824                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 825                         M_INTMOVE(s1, REG_ITMP1);
 826                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 827                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 828                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 829                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 830                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 831                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 832                         emit_store_dst(jd, iptr, d);
 833                         break;
 834
 835
 836                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 837
 838                         s1 = emit_load_s1(jd, iptr, RAX);
 839                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 840                         d = codegen_reg_of_dst(jd, iptr, RAX);
 841
 842                         M_INTMOVE(s1, RAX);
 843                         M_INTMOVE(s2, REG_ITMP3);
 844                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 845
 846                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 847
 848                         /* check as described in jvm spec */
 849                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 850                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 851                         M_BNE(4 + 6);
 852                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 853                         M_BEQ(2 + 3);                                          /* 6 bytes */
 854
 855                         emit_cqto(cd);                                         /* 2 bytes */
 856                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 857
 858                         M_INTMOVE(RAX, d);
 859                         emit_store_dst(jd, iptr, d);
 860                         dst = VAROP(iptr->dst);
 861                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 862                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 863                         break;
 864
 865                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 866
 867                         s1 = emit_load_s1(jd, iptr, RAX);
 868                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 869                         d = codegen_reg_of_dst(jd, iptr, RDX);
 870
 871                         M_INTMOVE(s1, RAX);
 872                         M_INTMOVE(s2, REG_ITMP3);
 873                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 874
 875                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 876
 877                         /* check as described in jvm spec */
 878                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 879                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 880                         M_BNE(3 + 4 + 6);
 881                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 882                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 883                         M_BEQ(2 + 3);                                          /* 6 bytes */
 884
 885                         emit_cqto(cd);                                         /* 2 bytes */
 886                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 887
 888                         M_INTMOVE(RDX, d);
 889                         emit_store_dst(jd, iptr, d);
 890                         dst = VAROP(iptr->dst);
 891                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 892                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 893                         break;
 894
 895                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 896                                       /* sx.val.i = constant                             */
 897
 898                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 899                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 900                         M_INTMOVE(s1, REG_ITMP1);
 901                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 902                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 903                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 904                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 905                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 906                         emit_store_dst(jd, iptr, d);
 907                         break;
 908
 909                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 910                                       /* sx.val.l = constant                             */
 911
 912                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 913                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 914                         M_INTMOVE(s1, REG_ITMP1);
 915                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 916                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 917                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 918                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 919                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 920                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 921                         emit_store_dst(jd, iptr, d);
 922                         break;
 923
 924                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 925
 926                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 927                         emit_ishift(jd, SHIFT_SHL, iptr);
 928                         break;
 929
 930                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 931                                       /* sx.val.i = constant                             */
 932
 933                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 934                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 935                         M_INTMOVE(s1, d);
 936                         M_ISLL_IMM(iptr->sx.val.i, d);
 937                         emit_store_dst(jd, iptr, d);
 938                         break;
 939
 940                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 941
 942                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 943                         emit_ishift(jd, SHIFT_SAR, iptr);
 944                         break;
 945
 946                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 947                                       /* sx.val.i = constant                             */
 948
 949                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 950                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 951                         M_INTMOVE(s1, d);
 952                         M_ISRA_IMM(iptr->sx.val.i, d);
 953                         emit_store_dst(jd, iptr, d);
 954                         break;
 955
 956                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 957
 958                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 959                         emit_ishift(jd, SHIFT_SHR, iptr);
 960                         break;
 961
 962                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 963                                       /* sx.val.i = constant                             */
 964
 965                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 966                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 967                         M_INTMOVE(s1, d);
 968                         M_ISRL_IMM(iptr->sx.val.i, d);
 969                         emit_store_dst(jd, iptr, d);
 970                         break;
 971
 972                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 973
 974                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 975                         emit_lshift(jd, SHIFT_SHL, iptr);
 976                         break;
 977
 978         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 979                                           /* sx.val.i = constant                             */
 980
 981                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 982                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 983                         M_INTMOVE(s1, d);
 984                         M_LSLL_IMM(iptr->sx.val.i, d);
 985                         emit_store_dst(jd, iptr, d);
 986                         break;
 987
 988                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 989
 990                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 991                         emit_lshift(jd, SHIFT_SAR, iptr);
 992                         break;
 993
 994                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 995                                       /* sx.val.i = constant                             */
 996
 997                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 998                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 999                         M_INTMOVE(s1, d);
1000                         M_LSRA_IMM(iptr->sx.val.i, d);
1001                         emit_store_dst(jd, iptr, d);
1002                         break;
1003
1004                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1005
1006                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1007                         emit_lshift(jd, SHIFT_SHR, iptr);
1008                         break;
1009
1010                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1011                                       /* sx.val.l = constant                             */
1012
1013                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1014                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1015                         M_INTMOVE(s1, d);
1016                         M_LSRL_IMM(iptr->sx.val.i, d);
1017                         emit_store_dst(jd, iptr, d);
1018                         break;
1019
1020                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1021
1022                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1023                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1024                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1025                         if (s2 == d)
1026                                 M_IAND(s1, d);
1027                         else {
1028                                 M_INTMOVE(s1, d);
1029                                 M_IAND(s2, d);
1030                         }
1031                         emit_store_dst(jd, iptr, d);
1032                         break;
1033
1034                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1035                                       /* sx.val.i = constant                             */
1036
1037                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1038                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1039                         M_INTMOVE(s1, d);
1040                         M_IAND_IMM(iptr->sx.val.i, d);
1041                         emit_store_dst(jd, iptr, d);
1042                         break;
1043
1044                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1045
1046                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1047                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1048                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1049                         if (s2 == d)
1050                                 M_LAND(s1, d);
1051                         else {
1052                                 M_INTMOVE(s1, d);
1053                                 M_LAND(s2, d);
1054                         }
1055                         emit_store_dst(jd, iptr, d);
1056                         break;
1057
1058                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1059                                       /* sx.val.l = constant                             */
1060
1061                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1062                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1063                         M_INTMOVE(s1, d);
1064                         if (IS_IMM32(iptr->sx.val.l))
1065                                 M_LAND_IMM(iptr->sx.val.l, d);
1066                         else {
1067                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1068                                 M_LAND(REG_ITMP2, d);
1069                         }
1070                         emit_store_dst(jd, iptr, d);
1071                         break;
1072
1073                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1074
1075                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1076                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1077                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1078                         if (s2 == d)
1079                                 M_IOR(s1, d);
1080                         else {
1081                                 M_INTMOVE(s1, d);
1082                                 M_IOR(s2, d);
1083                         }
1084                         emit_store_dst(jd, iptr, d);
1085                         break;
1086
1087                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1088                                       /* sx.val.i = constant                             */
1089
1090                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1091                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1092                         M_INTMOVE(s1, d);
1093                         M_IOR_IMM(iptr->sx.val.i, d);
1094                         emit_store_dst(jd, iptr, d);
1095                         break;
1096
1097                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1098
1099                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1100                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1101                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1102                         if (s2 == d)
1103                                 M_LOR(s1, d);
1104                         else {
1105                                 M_INTMOVE(s1, d);
1106                                 M_LOR(s2, d);
1107                         }
1108                         emit_store_dst(jd, iptr, d);
1109                         break;
1110
1111                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1112                                       /* sx.val.l = constant                             */
1113
1114                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1115                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1116                         M_INTMOVE(s1, d);
1117                         if (IS_IMM32(iptr->sx.val.l))
1118                                 M_LOR_IMM(iptr->sx.val.l, d);
1119                         else {
1120                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1121                                 M_LOR(REG_ITMP2, d);
1122                         }
1123                         emit_store_dst(jd, iptr, d);
1124                         break;
1125
1126                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1127
1128                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1129                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1130                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1131                         if (s2 == d)
1132                                 M_IXOR(s1, d);
1133                         else {
1134                                 M_INTMOVE(s1, d);
1135                                 M_IXOR(s2, d);
1136                         }
1137                         emit_store_dst(jd, iptr, d);
1138                         break;
1139
1140                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1141                                       /* sx.val.i = constant                             */
1142
1143                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1144                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1145                         M_INTMOVE(s1, d);
1146                         M_IXOR_IMM(iptr->sx.val.i, d);
1147                         emit_store_dst(jd, iptr, d);
1148                         break;
1149
1150                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1151
1152                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1153                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1154                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1155                         if (s2 == d)
1156                                 M_LXOR(s1, d);
1157                         else {
1158                                 M_INTMOVE(s1, d);
1159                                 M_LXOR(s2, d);
1160                         }
1161                         emit_store_dst(jd, iptr, d);
1162                         break;
1163
1164                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1165                                       /* sx.val.l = constant                             */
1166
1167                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1168                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1169                         M_INTMOVE(s1, d);
1170                         if (IS_IMM32(iptr->sx.val.l))
1171                                 M_LXOR_IMM(iptr->sx.val.l, d);
1172                         else {
1173                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1174                                 M_LXOR(REG_ITMP2, d);
1175                         }
1176                         emit_store_dst(jd, iptr, d);
1177                         break;
1178
1179
1180                 /* floating operations ************************************************/
1181
1182                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1183
1184                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1185                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1186                         disp = dseg_add_s4(cd, 0x80000000);
1187                         M_FLTMOVE(s1, d);
1188                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1189                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1190                         emit_store_dst(jd, iptr, d);
1191                         break;
1192
1193                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1194
1195                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1196                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1197                         disp = dseg_add_s8(cd, 0x8000000000000000);
1198                         M_FLTMOVE(s1, d);
1199                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1200                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1201                         emit_store_dst(jd, iptr, d);
1202                         break;
1203
1204                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1205
1206                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1207                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1208                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1209                         if (s2 == d)
1210                                 M_FADD(s1, d);
1211                         else {
1212                                 M_FLTMOVE(s1, d);
1213                                 M_FADD(s2, d);
1214                         }
1215                         emit_store_dst(jd, iptr, d);
1216                         break;
1217
1218                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1219
1220                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1221                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1222                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1223                         if (s2 == d)
1224                                 M_DADD(s1, d);
1225                         else {
1226                                 M_FLTMOVE(s1, d);
1227                                 M_DADD(s2, d);
1228                         }
1229                         emit_store_dst(jd, iptr, d);
1230                         break;
1231
1232                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1233
1234                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1235                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1236                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1237                         if (s2 == d) {
1238                                 M_FLTMOVE(s2, REG_FTMP2);
1239                                 s2 = REG_FTMP2;
1240                         }
1241                         M_FLTMOVE(s1, d);
1242                         M_FSUB(s2, d);
1243                         emit_store_dst(jd, iptr, d);
1244                         break;
1245
1246                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1247
1248                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1249                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1250                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1251                         if (s2 == d) {
1252                                 M_FLTMOVE(s2, REG_FTMP2);
1253                                 s2 = REG_FTMP2;
1254                         }
1255                         M_FLTMOVE(s1, d);
1256                         M_DSUB(s2, d);
1257                         emit_store_dst(jd, iptr, d);
1258                         break;
1259
1260                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1261
1262                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1263                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1264                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1265                         if (s2 == d)
1266                                 M_FMUL(s1, d);
1267                         else {
1268                                 M_FLTMOVE(s1, d);
1269                                 M_FMUL(s2, d);
1270                         }
1271                         emit_store_dst(jd, iptr, d);
1272                         break;
1273
1274                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1275
1276                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1277                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1278                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1279                         if (s2 == d)
1280                                 M_DMUL(s1, d);
1281                         else {
1282                                 M_FLTMOVE(s1, d);
1283                                 M_DMUL(s2, d);
1284                         }
1285                         emit_store_dst(jd, iptr, d);
1286                         break;
1287
1288                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1289
1290                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1291                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1292                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1293                         if (s2 == d) {
1294                                 M_FLTMOVE(s2, REG_FTMP2);
1295                                 s2 = REG_FTMP2;
1296                         }
1297                         M_FLTMOVE(s1, d);
1298                         M_FDIV(s2, d);
1299                         emit_store_dst(jd, iptr, d);
1300                         break;
1301
1302                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1303
1304                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1305                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1306                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1307                         if (s2 == d) {
1308                                 M_FLTMOVE(s2, REG_FTMP2);
1309                                 s2 = REG_FTMP2;
1310                         }
1311                         M_FLTMOVE(s1, d);
1312                         M_DDIV(s2, d);
1313                         emit_store_dst(jd, iptr, d);
1314                         break;
1315
1316                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1317
1318                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1319                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1320                         M_CVTIF(s1, d);
1321                         emit_store_dst(jd, iptr, d);
1322                         break;
1323
1324                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1325
1326                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1327                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1328                         M_CVTID(s1, d);
1329                         emit_store_dst(jd, iptr, d);
1330                         break;
1331
1332                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1333
1334                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1335                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1336                         M_CVTLF(s1, d);
1337                         emit_store_dst(jd, iptr, d);
1338                         break;
1339
1340                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1341
1342                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1343                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1344                         M_CVTLD(s1, d);
1345                         emit_store_dst(jd, iptr, d);
1346                         break;
1347
1348                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1349
1350                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1351                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1352                         M_CVTFI(s1, d);
1353                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1354                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1355                                 ((REG_RESULT == d) ? 0 : 3);
1356                         M_BNE(disp);
1357                         M_FLTMOVE(s1, REG_FTMP1);
1358                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1359                         M_CALL(REG_ITMP2);
1360                         M_INTMOVE(REG_RESULT, d);
1361                         emit_store_dst(jd, iptr, d);
1362                         break;
1363
1364                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1365
1366                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1367                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1368                         M_CVTDI(s1, d);
1369                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1370                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1371                                 ((REG_RESULT == d) ? 0 : 3);
1372                         M_BNE(disp);
1373                         M_FLTMOVE(s1, REG_FTMP1);
1374                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1375                         M_CALL(REG_ITMP2);
1376                         M_INTMOVE(REG_RESULT, d);
1377                         emit_store_dst(jd, iptr, d);
1378                         break;
1379
1380                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1381
1382                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1383                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1384                         M_CVTFL(s1, d);
1385                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1386                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1387                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1388                                 ((REG_RESULT == d) ? 0 : 3);
1389                         M_BNE(disp);
1390                         M_FLTMOVE(s1, REG_FTMP1);
1391                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1392                         M_CALL(REG_ITMP2);
1393                         M_INTMOVE(REG_RESULT, d);
1394                         emit_store_dst(jd, iptr, d);
1395                         break;
1396
1397                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1398
1399                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1400                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1401                         M_CVTDL(s1, d);
1402                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1403                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1404                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1405                                 ((REG_RESULT == d) ? 0 : 3);
1406                         M_BNE(disp);
1407                         M_FLTMOVE(s1, REG_FTMP1);
1408                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1409                         M_CALL(REG_ITMP2);
1410                         M_INTMOVE(REG_RESULT, d);
1411                         emit_store_dst(jd, iptr, d);
1412                         break;
1413
1414                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1415
1416                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1417                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1418                         M_CVTFD(s1, d);
1419                         emit_store_dst(jd, iptr, d);
1420                         break;
1421
1422                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1423
1424                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1425                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1426                         M_CVTDF(s1, d);
1427                         emit_store_dst(jd, iptr, d);
1428                         break;
1429
1430                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1431                                           /* == => 0, < => 1, > => -1 */
1432
1433                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1434                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1435                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1436                         M_CLR(d);
1437                         M_MOV_IMM(1, REG_ITMP1);
1438                         M_MOV_IMM(-1, REG_ITMP2);
1439                         emit_ucomiss_reg_reg(cd, s1, s2);
1440                         M_CMOVULT(REG_ITMP1, d);
1441                         M_CMOVUGT(REG_ITMP2, d);
1442                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1443                         emit_store_dst(jd, iptr, d);
1444                         break;
1445
1446                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1447                                           /* == => 0, < => 1, > => -1 */
1448
1449                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1450                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1451                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1452                         M_CLR(d);
1453                         M_MOV_IMM(1, REG_ITMP1);
1454                         M_MOV_IMM(-1, REG_ITMP2);
1455                         emit_ucomiss_reg_reg(cd, s1, s2);
1456                         M_CMOVULT(REG_ITMP1, d);
1457                         M_CMOVUGT(REG_ITMP2, d);
1458                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1459                         emit_store_dst(jd, iptr, d);
1460                         break;
1461
1462                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1463                                           /* == => 0, < => 1, > => -1 */
1464
1465                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1466                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1467                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1468                         M_CLR(d);
1469                         M_MOV_IMM(1, REG_ITMP1);
1470                         M_MOV_IMM(-1, REG_ITMP2);
1471                         emit_ucomisd_reg_reg(cd, s1, s2);
1472                         M_CMOVULT(REG_ITMP1, d);
1473                         M_CMOVUGT(REG_ITMP2, d);
1474                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1475                         emit_store_dst(jd, iptr, d);
1476                         break;
1477
1478                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1479                                           /* == => 0, < => 1, > => -1 */
1480
1481                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1482                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1483                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1484                         M_CLR(d);
1485                         M_MOV_IMM(1, REG_ITMP1);
1486                         M_MOV_IMM(-1, REG_ITMP2);
1487                         emit_ucomisd_reg_reg(cd, s1, s2);
1488                         M_CMOVULT(REG_ITMP1, d);
1489                         M_CMOVUGT(REG_ITMP2, d);
1490                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1491                         emit_store_dst(jd, iptr, d);
1492                         break;
1493
1494
1495                 /* memory operations **************************************************/
1496
1497                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1498
1499                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1500                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1501                         /* implicit null-pointer check */
1502                         M_ILD(d, s1, OFFSET(java_array_t, size));
1503                         emit_store_dst(jd, iptr, d);
1504                         break;
1505
1506                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1507
1508                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1509                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1510                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1511                         /* implicit null-pointer check */
1512                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1513                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1514                         emit_store_dst(jd, iptr, d);
1515                         break;
1516
1517                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1518
1519                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1520                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1521                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1522                         /* implicit null-pointer check */
1523                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1524                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1525                         emit_store_dst(jd, iptr, d);
1526                         break;
1527
1528                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1529
1530                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1531                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1532                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1533                         /* implicit null-pointer check */
1534                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1535                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1536                         emit_store_dst(jd, iptr, d);
1537                         break;
1538
1539                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1540
1541                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1542                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1543                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1544                         /* implicit null-pointer check */
1545                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1546                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1547                         emit_store_dst(jd, iptr, d);
1548                         break;
1549
1550                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1551
1552                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1553                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1554                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1555                         /* implicit null-pointer check */
1556                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1557                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1558                         emit_store_dst(jd, iptr, d);
1559                         break;
1560
1561                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1562
1563                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1564                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1565                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1566                         /* implicit null-pointer check */
1567                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1568                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1569                         emit_store_dst(jd, iptr, d);
1570                         break;
1571
1572                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1573
1574                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1575                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1576                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1577                         /* implicit null-pointer check */
1578                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1579                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1580                         emit_store_dst(jd, iptr, d);
1581                         break;
1582
1583                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1584
1585                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1586                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1587                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1588                         /* implicit null-pointer check */
1589                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1590                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1591                         emit_store_dst(jd, iptr, d);
1592                         break;
1593
1594
1595                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1596
1597                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1598                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1599                         /* implicit null-pointer check */
1600                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1601                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1602                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1603                         break;
1604
1605                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1606
1607                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1608                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1609                         /* implicit null-pointer check */
1610                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1611                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1612                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1613                         break;
1614
1615                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1616
1617                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1618                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1619                         /* implicit null-pointer check */
1620                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1621                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1622                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1623                         break;
1624
1625                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1626
1627                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1628                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1629                         /* implicit null-pointer check */
1630                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1631                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1632                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1633                         break;
1634
1635                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1636
1637                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1638                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1639                         /* implicit null-pointer check */
1640                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1641                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1642                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1643                         break;
1644
1645                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1646
1647                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1648                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1649                         /* implicit null-pointer check */
1650                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1651                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1652                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1653                         break;
1654
1655                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1656
1657                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1658                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1659                         /* implicit null-pointer check */
1660                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1661                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1662                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1663                         break;
1664
1665                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1666
1667                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1668                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1669                         /* implicit null-pointer check */
1670                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1671                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1672
1673                         M_MOV(s1, REG_A0);
1674                         M_MOV(s3, REG_A1);
1675                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
1676                         M_CALL(REG_ITMP1);
1677                         emit_exception_check(cd, iptr);
1678
1679                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1680                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1681                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1682                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1683                         break;
1684
1685
1686                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1687
1688                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1689                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1690                         /* implicit null-pointer check */
1691                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1692                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1693                         break;
1694
1695                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1696
1697                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1698                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1699                         /* implicit null-pointer check */
1700                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1701                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1702                         break;
1703
1704                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1705
1706                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1707                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1708                         /* implicit null-pointer check */
1709                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1710                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1711                         break;
1712
1713                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1714
1715                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1716                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1717                         /* implicit null-pointer check */
1718                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1719                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1720                         break;
1721
1722                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1723
1724                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1725                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1726                         /* implicit null-pointer check */
1727                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1728
1729                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1730                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1731                         }
1732                         else {
1733                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1734                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1735                         }
1736                         break;
1737
1738                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1739
1740                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1741                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1742                         /* implicit null-pointer check */
1743                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1744                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1745                         break;
1746
1747
1748                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1749
1750                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1751                                 uf        = iptr->sx.s23.s3.uf;
1752                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1753                                 disp      = dseg_add_unique_address(cd, NULL);
1754                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1755
1756                                 /* must be calculated before codegen_add_patch_ref */
1757
1758                                 if (opt_shownops)
1759                                         disp -= PATCHER_CALL_SIZE;
1760
1761 /*                              PROFILE_CYCLE_STOP; */
1762
1763                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1764
1765 /*                              PROFILE_CYCLE_START; */
1766                         }
1767                         else {
1768                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1769                                 fieldtype = fi->type;
1770                                 disp      = dseg_add_address(cd, fi->value);
1771                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1772
1773                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1774                                         PROFILE_CYCLE_STOP;
1775
1776                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1777
1778                                         PROFILE_CYCLE_START;
1779
1780                                         if (opt_shownops)
1781                                                 disp -= PATCHER_CALL_SIZE;
1782                                 }
1783                         }
1784
1785                         /* This approach is much faster than moving the field
1786                            address inline into a register. */
1787
1788                         M_ALD(REG_ITMP1, RIP, disp);
1789
1790                         switch (fieldtype) {
1791                         case TYPE_INT:
1792                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1793                                 M_ILD(d, REG_ITMP1, 0);
1794                                 break;
1795                         case TYPE_LNG:
1796                         case TYPE_ADR:
1797                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1798                                 M_LLD(d, REG_ITMP1, 0);
1799                                 break;
1800                         case TYPE_FLT:
1801                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1802                                 M_FLD(d, REG_ITMP1, 0);
1803                                 break;
1804                         case TYPE_DBL:
1805                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1806                                 M_DLD(d, REG_ITMP1, 0);
1807                                 break;
1808                         }
1809                         emit_store_dst(jd, iptr, d);
1810                         break;
1811
1812                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1813
1814                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1815                                 uf        = iptr->sx.s23.s3.uf;
1816                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1817                                 disp      = dseg_add_unique_address(cd, NULL);
1818                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1819
1820                                 /* must be calculated before codegen_add_patch_ref */
1821
1822                                 if (opt_shownops)
1823                                         disp -= PATCHER_CALL_SIZE;
1824
1825 /*                              PROFILE_CYCLE_STOP; */
1826
1827                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1828
1829 /*                              PROFILE_CYCLE_START; */
1830                         }
1831                         else {
1832                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1833                                 fieldtype = fi->type;
1834                                 disp      = dseg_add_address(cd, fi->value);
1835                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1836
1837                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1838                                         PROFILE_CYCLE_STOP;
1839
1840                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1841
1842                                         PROFILE_CYCLE_START;
1843
1844                                         if (opt_shownops)
1845                                                 disp -= PATCHER_CALL_SIZE;
1846                                 }
1847                         }
1848
1849                         /* This approach is much faster than moving the field
1850                            address inline into a register. */
1851
1852                         M_ALD(REG_ITMP1, RIP, disp);
1853
1854                         switch (fieldtype) {
1855                         case TYPE_INT:
1856                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1857                                 M_IST(s1, REG_ITMP1, 0);
1858                                 break;
1859                         case TYPE_LNG:
1860                         case TYPE_ADR:
1861                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1862                                 M_LST(s1, REG_ITMP1, 0);
1863                                 break;
1864                         case TYPE_FLT:
1865                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1866                                 M_FST(s1, REG_ITMP1, 0);
1867                                 break;
1868                         case TYPE_DBL:
1869                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1870                                 M_DST(s1, REG_ITMP1, 0);
1871                                 break;
1872                         }
1873                         break;
1874
1875                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1876                                           /* val = value (in current instruction)     */
1877                                           /* following NOP)                           */
1878
1879                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1880                                 uf        = iptr->sx.s23.s3.uf;
1881                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1882                                 disp      = dseg_add_unique_address(cd, NULL);
1883                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1884
1885                                 /* must be calculated before codegen_add_patch_ref */
1886
1887                                 if (opt_shownops)
1888                                         disp -= PATCHER_CALL_SIZE;
1889
1890 /*                              PROFILE_CYCLE_STOP; */
1891
1892                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1893
1894 /*                              PROFILE_CYCLE_START; */
1895                         }
1896                         else {
1897                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1898                                 fieldtype = fi->type;
1899                                 disp      = dseg_add_address(cd, fi->value);
1900                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1901
1902                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1903                                         PROFILE_CYCLE_STOP;
1904
1905                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1906
1907                                         PROFILE_CYCLE_START;
1908
1909                                         if (opt_shownops)
1910                                                 disp -= PATCHER_CALL_SIZE;
1911                                 }
1912                         }
1913
1914                         /* This approach is much faster than moving the field
1915                            address inline into a register. */
1916
1917                         M_ALD(REG_ITMP1, RIP, disp);
1918
1919                         switch (fieldtype) {
1920                         case TYPE_INT:
1921                         case TYPE_FLT:
1922                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1923                                 break;
1924                         case TYPE_LNG:
1925                         case TYPE_ADR:
1926                         case TYPE_DBL:
1927                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1928                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1929                                 else {
1930                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1931                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1932                                 }
1933                                 break;
1934                         }
1935                         break;
1936
1937                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1938
1939                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1940
1941                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1942                                 uf        = iptr->sx.s23.s3.uf;
1943                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1944                                 disp      = 0;
1945
1946 /*                              PROFILE_CYCLE_STOP; */
1947
1948                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1949
1950 /*                              PROFILE_CYCLE_START; */
1951                         }
1952                         else {
1953                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1954                                 fieldtype = fi->type;
1955                                 disp      = fi->offset;
1956                         }
1957
1958                         /* implicit null-pointer check */
1959                         switch (fieldtype) {
1960                         case TYPE_INT:
1961                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1962                                 M_ILD32(d, s1, disp);
1963                                 break;
1964                         case TYPE_LNG:
1965                         case TYPE_ADR:
1966                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1967                                 M_LLD32(d, s1, disp);
1968                                 break;
1969                         case TYPE_FLT:
1970                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1971                                 M_FLD32(d, s1, disp);
1972                                 break;
1973                         case TYPE_DBL:
1974                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1975                                 M_DLD32(d, s1, disp);
1976                                 break;
1977                         }
1978                         emit_store_dst(jd, iptr, d);
1979                         break;
1980
1981                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1982
1983                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1984                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1985
1986                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1987                                 uf        = iptr->sx.s23.s3.uf;
1988                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1989                                 disp      = 0;
1990
1991 /*                              PROFILE_CYCLE_STOP; */
1992
1993                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1994
1995 /*                              PROFILE_CYCLE_START; */
1996                         }
1997                         else {
1998                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1999                                 fieldtype = fi->type;
2000                                 disp      = fi->offset;
2001                         }
2002
2003                         /* implicit null-pointer check */
2004                         switch (fieldtype) {
2005                         case TYPE_INT:
2006                                 M_IST32(s2, s1, disp);
2007                                 break;
2008                         case TYPE_LNG:
2009                         case TYPE_ADR:
2010                                 M_LST32(s2, s1, disp);
2011                                 break;
2012                         case TYPE_FLT:
2013                                 M_FST32(s2, s1, disp);
2014                                 break;
2015                         case TYPE_DBL:
2016                                 M_DST32(s2, s1, disp);
2017                                 break;
2018                         }
2019                         break;
2020
2021                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2022                                           /* val = value (in current instruction)     */
2023                                           /* following NOP)                           */
2024
2025                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2026
2027                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2028                                 uf        = iptr->sx.s23.s3.uf;
2029                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2030                                 disp      = 0;
2031
2032 /*                              PROFILE_CYCLE_STOP; */
2033
2034                                 codegen_add_patch_ref(cd, PATCHER_putfieldconst, uf, 0);
2035
2036 /*                              PROFILE_CYCLE_START; */
2037                         }
2038                         else {
2039                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2040                                 fieldtype = fi->type;
2041                                 disp      = fi->offset;
2042                         }
2043
2044                         /* implicit null-pointer check */
2045                         switch (fieldtype) {
2046                         case TYPE_INT:
2047                         case TYPE_FLT:
2048                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2049                                 break;
2050                         case TYPE_LNG:
2051                         case TYPE_ADR:
2052                         case TYPE_DBL:
2053                                 /* XXX why no check for IS_IMM32? */
2054                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2055                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2056                                 break;
2057                         }
2058                         break;
2059
2060
2061                 /* branch operations **************************************************/
2062
2063                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2064
2065                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2066                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2067
2068                         PROFILE_CYCLE_STOP;
2069
2070 #ifdef ENABLE_VERIFIER
2071                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2072                                 uc = iptr->sx.s23.s2.uc;
2073
2074                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2075                         }
2076 #endif /* ENABLE_VERIFIER */
2077
2078                         M_CALL_IMM(0);                            /* passing exception pc */
2079                         M_POP(REG_ITMP2_XPC);
2080
2081                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2082                         M_JMP(REG_ITMP3);
2083                         break;
2084
2085                 case ICMD_GOTO:         /* ... ==> ...                                */
2086                 case ICMD_RET:
2087
2088                         emit_br(cd, iptr->dst.block);
2089                         ALIGNCODENOP;
2090                         break;
2091
2092                 case ICMD_JSR:          /* ... ==> ...                                */
2093
2094                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2095                         ALIGNCODENOP;
2096                         break;
2097
2098                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2099                 case ICMD_IFNONNULL:
2100
2101                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2102                         M_TEST(s1);
2103                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2104                         break;
2105
2106                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2107                 case ICMD_IFLT:
2108                 case ICMD_IFLE:
2109                 case ICMD_IFNE:
2110                 case ICMD_IFGT:
2111                 case ICMD_IFGE:
2112
2113                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2114                         M_ICMP_IMM(iptr->sx.val.i, s1);
2115                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2116                         break;
2117
2118                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2119                 case ICMD_IF_LNE:
2120                 case ICMD_IF_LLT:
2121                 case ICMD_IF_LGE:
2122                 case ICMD_IF_LGT:
2123                 case ICMD_IF_LLE:
2124
2125                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2126                         if (IS_IMM32(iptr->sx.val.l))
2127                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2128                         else {
2129                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2130                                 M_LCMP(REG_ITMP2, s1);
2131                         }
2132                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2133                         break;
2134
2135                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2136                 case ICMD_IF_ICMPNE:
2137                 case ICMD_IF_ICMPLT:
2138                 case ICMD_IF_ICMPGE:
2139                 case ICMD_IF_ICMPGT:
2140                 case ICMD_IF_ICMPLE:
2141
2142                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2143                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2144                         M_ICMP(s2, s1);
2145                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2146                         break;
2147
2148                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2149                 case ICMD_IF_ACMPNE:
2150
2151                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2152                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2153                         M_LCMP(s2, s1);
2154                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2155                         break;
2156
2157                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2158                 case ICMD_IF_LCMPNE:
2159                 case ICMD_IF_LCMPLT:
2160                 case ICMD_IF_LCMPGE:
2161                 case ICMD_IF_LCMPGT:
2162                 case ICMD_IF_LCMPLE:
2163
2164                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2165                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2166                         M_LCMP(s2, s1);
2167                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2168                         break;
2169
2170                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2171                 case ICMD_LRETURN:
2172
2173                         REPLACEMENT_POINT_RETURN(cd, iptr);
2174                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2175                         M_INTMOVE(s1, REG_RESULT);
2176                         goto nowperformreturn;
2177
2178                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2179
2180                         REPLACEMENT_POINT_RETURN(cd, iptr);
2181                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2182                         M_INTMOVE(s1, REG_RESULT);
2183
2184 #ifdef ENABLE_VERIFIER
2185                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2186                                 uc = iptr->sx.s23.s2.uc;
2187
2188                                 PROFILE_CYCLE_STOP;
2189
2190                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2191
2192                                 PROFILE_CYCLE_START;
2193                         }
2194 #endif /* ENABLE_VERIFIER */
2195                         goto nowperformreturn;
2196
2197                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2198                 case ICMD_DRETURN:
2199
2200                         REPLACEMENT_POINT_RETURN(cd, iptr);
2201                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2202                         M_FLTMOVE(s1, REG_FRESULT);
2203                         goto nowperformreturn;
2204
2205                 case ICMD_RETURN:      /* ...  ==> ...                                */
2206
2207                         REPLACEMENT_POINT_RETURN(cd, iptr);
2208
2209 nowperformreturn:
2210                         {
2211                         s4 i, p;
2212
2213                         p = cd->stackframesize;
2214
2215 #if !defined(NDEBUG)
2216                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2217                                 emit_verbosecall_exit(jd);
2218 #endif /* !defined(NDEBUG) */
2219
2220 #if defined(ENABLE_THREADS)
2221                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2222                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2223
2224                                 /* we need to save the proper return value */
2225                                 switch (iptr->opc) {
2226                                 case ICMD_IRETURN:
2227                                 case ICMD_ARETURN:
2228                                 case ICMD_LRETURN:
2229                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2230                                         break;
2231                                 case ICMD_FRETURN:
2232                                 case ICMD_DRETURN:
2233                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2234                                         break;
2235                                 }
2236
2237                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2238                                 M_CALL(REG_ITMP1);
2239
2240                                 /* and now restore the proper return value */
2241                                 switch (iptr->opc) {
2242                                 case ICMD_IRETURN:
2243                                 case ICMD_ARETURN:
2244                                 case ICMD_LRETURN:
2245                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2246                                         break;
2247                                 case ICMD_FRETURN:
2248                                 case ICMD_DRETURN:
2249                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2250                                         break;
2251                                 }
2252                         }
2253 #endif
2254
2255                         /* restore saved registers */
2256
2257                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2258                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2259                         }
2260                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2261                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2262                         }
2263
2264                         /* deallocate stack */
2265
2266                         if (cd->stackframesize)
2267                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2268
2269                         /* generate method profiling code */
2270
2271                         PROFILE_CYCLE_STOP;
2272
2273                         M_RET;
2274                         }
2275                         break;
2276
2277
2278                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2279                         {
2280                                 s4 i, l;
2281                                 branch_target_t *table;
2282
2283                                 table = iptr->dst.table;
2284
2285                                 l = iptr->sx.s23.s2.tablelow;
2286                                 i = iptr->sx.s23.s3.tablehigh;
2287
2288                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2289                                 M_INTMOVE(s1, REG_ITMP1);
2290
2291                                 if (l != 0)
2292                                         M_ISUB_IMM(l, REG_ITMP1);
2293
2294                                 /* number of targets */
2295                                 i = i - l + 1;
2296
2297                 /* range check */
2298
2299                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2300                                 emit_bugt(cd, table[0].block);
2301
2302                                 /* build jump table top down and use address of lowest entry */
2303
2304                                 table += i;
2305
2306                                 while (--i >= 0) {
2307                                         dseg_add_target(cd, table->block);
2308                                         --table;
2309                                 }
2310
2311                                 /* length of dataseg after last dseg_add_target is used
2312                                    by load */
2313
2314                                 M_MOV_IMM(0, REG_ITMP2);
2315                                 dseg_adddata(cd);
2316                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2317                                 M_JMP(REG_ITMP1);
2318                         }
2319                         break;
2320
2321
2322                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2323                         {
2324                                 s4 i;
2325                                 lookup_target_t *lookup;
2326
2327                                 lookup = iptr->dst.lookup;
2328
2329                                 i = iptr->sx.s23.s2.lookupcount;
2330
2331                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2332                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2333
2334                                 while (--i >= 0) {
2335                                         M_ICMP_IMM(lookup->value, s1);
2336                                         emit_beq(cd, lookup->target.block);
2337                                         lookup++;
2338                                 }
2339
2340                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2341                                 ALIGNCODENOP;
2342                         }
2343                         break;
2344
2345
2346                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2347
2348                         bte = iptr->sx.s23.s3.bte;
2349                         md  = bte->md;
2350                         goto gen_method;
2351
2352                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2353
2354                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2355                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2356                 case ICMD_INVOKEINTERFACE:
2357
2358                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2359
2360                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2361                                 lm = NULL;
2362                                 um = iptr->sx.s23.s3.um;
2363                                 md = um->methodref->parseddesc.md;
2364                         }
2365                         else {
2366                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2367                                 um = NULL;
2368                                 md = lm->parseddesc;
2369                         }
2370
2371 gen_method:
2372                         s3 = md->paramcount;
2373
2374                         MCODECHECK((20 * s3) + 128);
2375
2376                         /* copy arguments to registers or stack location */
2377
2378                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2379                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2380                                 d   = md->params[s3].regoff;
2381
2382                                 /* already preallocated (ARGVAR)? */
2383
2384                                 if (var->flags & PREALLOC)
2385                                         continue;
2386
2387                                 if (IS_INT_LNG_TYPE(var->type)) {
2388                                         if (!md->params[s3].inmemory) {
2389                                                 s1 = emit_load(jd, iptr, var, d);
2390                                                 M_INTMOVE(s1, d);
2391                                         }
2392                                         else {
2393                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2394                                                 M_LST(s1, REG_SP, d);
2395                                         }
2396                                 }
2397                                 else {
2398                                         if (!md->params[s3].inmemory) {
2399                                                 s1 = emit_load(jd, iptr, var, d);
2400                                                 M_FLTMOVE(s1, d);
2401                                         }
2402                                         else {
2403                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2404
2405                                                 if (IS_2_WORD_TYPE(var->type))
2406                                                         M_DST(s1, REG_SP, d);
2407                                                 else
2408                                                         M_FST(s1, REG_SP, d);
2409                                         }
2410                                 }
2411                         }
2412
2413                         /* generate method profiling code */
2414
2415                         PROFILE_CYCLE_STOP;
2416
2417                         switch (iptr->opc) {
2418                         case ICMD_BUILTIN:
2419                                 M_MOV_IMM(bte->fp, REG_ITMP1);
2420                                 M_CALL(REG_ITMP1);
2421
2422                                 emit_exception_check(cd, iptr);
2423                                 break;
2424
2425                         case ICMD_INVOKESPECIAL:
2426                                 emit_nullpointer_check(cd, iptr, REG_A0);
2427                                 /* fall through */
2428
2429                         case ICMD_INVOKESTATIC:
2430                                 if (lm == NULL) {
2431                                         disp = dseg_add_unique_address(cd, NULL);
2432                                         disp = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
2433
2434                                         /* must be calculated before codegen_add_patch_ref */
2435
2436                                         if (opt_shownops)
2437                                                 disp -= PATCHER_CALL_SIZE;
2438
2439                                         codegen_add_patch_ref(cd, PATCHER_invokestatic_special,
2440                                                                                   um, disp);
2441
2442 /*                                      a = 0; */
2443                                 }
2444                                 else {
2445                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2446                                         disp = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
2447
2448 /*                                      a = (ptrint) lm->stubroutine; */
2449                                 }
2450
2451 /*                              M_MOV_IMM(a, REG_ITMP2); */
2452                                 M_ALD(REG_ITMP2, RIP, disp);
2453                                 M_CALL(REG_ITMP2);
2454                                 break;
2455
2456                         case ICMD_INVOKEVIRTUAL:
2457                                 if (lm == NULL) {
2458                                         codegen_add_patch_ref(cd, PATCHER_invokevirtual, um, 0);
2459
2460                                         s1 = 0;
2461                                 }
2462                                 else {
2463                                         s1 = OFFSET(vftbl_t, table[0]) +
2464                                                 sizeof(methodptr) * lm->vftblindex;
2465                                 }
2466
2467                                 /* implicit null-pointer check */
2468                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2469                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2470                                 M_CALL(REG_ITMP3);
2471                                 break;
2472
2473                         case ICMD_INVOKEINTERFACE:
2474                                 if (lm == NULL) {
2475                                         codegen_add_patch_ref(cd, PATCHER_invokeinterface, um, 0);
2476
2477                                         s1 = 0;
2478                                         s2 = 0;
2479                                 }
2480                                 else {
2481                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2482                                                 sizeof(methodptr) * lm->class->index;
2483
2484                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2485                                 }
2486
2487                                 /* implicit null-pointer check */
2488                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2489                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2490                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2491                                 M_CALL(REG_ITMP3);
2492                                 break;
2493                         }
2494
2495                         /* generate method profiling code */
2496
2497                         PROFILE_CYCLE_START;
2498
2499                         /* store size of call code in replacement point */
2500
2501                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2502
2503                         /* store return value */
2504
2505                         switch (md->returntype.type) {
2506                         case TYPE_INT:
2507                         case TYPE_LNG:
2508                         case TYPE_ADR:
2509                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2510                                 M_INTMOVE(REG_RESULT, s1);
2511                                 emit_store_dst(jd, iptr, s1);
2512                                 break;
2513                         case TYPE_FLT:
2514                         case TYPE_DBL:
2515                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2516                                 M_FLTMOVE(REG_FRESULT, s1);
2517                                 emit_store_dst(jd, iptr, s1);
2518                                 break;
2519                         default:
2520                                 /* TYPE_VOID */
2521                                 break;
2522                         }
2523                         break;
2524
2525
2526                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2527
2528                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2529                                 /* object type cast-check */
2530
2531                                 classinfo *super;
2532                                 vftbl_t   *supervftbl;
2533                                 s4         superindex;
2534
2535                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2536                                         super      = NULL;
2537                                         superindex = 0;
2538                                         supervftbl = NULL;
2539                                 }
2540                                 else {
2541                                         super      = iptr->sx.s23.s3.c.cls;
2542                                         superindex = super->index;
2543                                         supervftbl = super->vftbl;
2544                                 }
2545
2546                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2547                                         CODEGEN_CRITICAL_SECTION_NEW;
2548
2549                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2550
2551                                 /* if class is not resolved, check which code to call */
2552
2553                                 if (super == NULL) {
2554                                         M_TEST(s1);
2555                                         emit_label_beq(cd, BRANCH_LABEL_1);
2556
2557                                         codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2558                                                                                   iptr->sx.s23.s3.c.ref, 0);
2559
2560                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2561                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2562                                         emit_label_beq(cd, BRANCH_LABEL_2);
2563                                 }
2564
2565                                 /* interface checkcast code */
2566
2567                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2568                                         if (super != NULL) {
2569                                                 M_TEST(s1);
2570                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2571                                         }
2572
2573                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2574
2575                                         if (super == NULL) {
2576                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_interface,
2577                                                                                           iptr->sx.s23.s3.c.ref,
2578                                                                                           0);
2579                                         }
2580
2581                                         M_ILD32(REG_ITMP3,
2582                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2583                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2584                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2585
2586                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2587                                                         OFFSET(vftbl_t, interfacetable[0]) -
2588                                                         superindex * sizeof(methodptr*));
2589                                         M_TEST(REG_ITMP3);
2590                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2591
2592                                         if (super == NULL)
2593                                                 emit_label_br(cd, BRANCH_LABEL_4);
2594                                         else
2595                                                 emit_label(cd, BRANCH_LABEL_3);
2596                                 }
2597
2598                                 /* class checkcast code */
2599
2600                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2601                                         if (super == NULL) {
2602                                                 emit_label(cd, BRANCH_LABEL_2);
2603                                         }
2604                                         else {
2605                                                 M_TEST(s1);
2606                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2607                                         }
2608
2609                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2610
2611                                         if (super == NULL) {
2612                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_class,
2613                                                                                           iptr->sx.s23.s3.c.ref,
2614                                                                                           0);
2615                                         }
2616
2617                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2618
2619                                         CODEGEN_CRITICAL_SECTION_START;
2620
2621                                         M_ILD32(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2622
2623                                         /*                                      if (s1 != REG_ITMP1) { */
2624                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2625                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2626                                         /*                                                                                              REG_ITMP1); */
2627                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2628                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2629                                         /*                                                                                              REG_ITMP3); */
2630                                         /*  #if defined(ENABLE_THREADS) */
2631                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2632                                         /*  #endif */
2633                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2634
2635                                         /*                                      } else { */
2636
2637                                         M_ILD32(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
2638                                         M_ISUB(REG_ITMP3, REG_ITMP2);
2639                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2640                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2641                                         /*                                      } */
2642
2643                                         CODEGEN_CRITICAL_SECTION_END;
2644
2645                                         M_ICMP(REG_ITMP3, REG_ITMP2);
2646                                         emit_classcast_check(cd, iptr, BRANCH_UGT, REG_ITMP3, s1);
2647
2648                                         if (super != NULL)
2649                                                 emit_label(cd, BRANCH_LABEL_5);
2650                                 }
2651
2652                                 if (super == NULL) {
2653                                         emit_label(cd, BRANCH_LABEL_1);
2654                                         emit_label(cd, BRANCH_LABEL_4);
2655                                 }
2656
2657                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2658                         }
2659                         else {
2660                                 /* array type cast-check */
2661
2662                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2663                                 M_INTMOVE(s1, REG_A0);
2664
2665                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2666                                         codegen_add_patch_ref(cd, PATCHER_builtin_arraycheckcast,
2667                                                                                   iptr->sx.s23.s3.c.ref, 0);
2668                                 }
2669
2670                                 M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2671                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2672                                 M_CALL(REG_ITMP1);
2673
2674                                 /* s1 may have been destroyed over the function call */
2675                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2676                                 M_TEST(REG_RESULT);
2677                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2678
2679                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2680                         }
2681
2682                         M_INTMOVE(s1, d);
2683                         emit_store_dst(jd, iptr, d);
2684                         break;
2685
2686                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2687
2688                         {
2689                         classinfo *super;
2690                         vftbl_t   *supervftbl;
2691                         s4         superindex;
2692
2693                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2694                                 super      = NULL;
2695                                 superindex = 0;
2696                                 supervftbl = NULL;
2697                         }
2698                         else {
2699                                 super      = iptr->sx.s23.s3.c.cls;
2700                                 superindex = super->index;
2701                                 supervftbl = super->vftbl;
2702                         }
2703
2704                         if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2705                                 CODEGEN_CRITICAL_SECTION_NEW;
2706
2707                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2708                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2709
2710                         if (s1 == d) {
2711                                 M_INTMOVE(s1, REG_ITMP1);
2712                                 s1 = REG_ITMP1;
2713                         }
2714
2715                         M_CLR(d);
2716
2717                         /* if class is not resolved, check which code to call */
2718
2719                         if (super == NULL) {
2720                                 M_TEST(s1);
2721                                 emit_label_beq(cd, BRANCH_LABEL_1);
2722
2723                                 codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2724                                                                           iptr->sx.s23.s3.c.ref, 0);
2725
2726                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2727                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2728                                 emit_label_beq(cd, BRANCH_LABEL_2);
2729                         }
2730
2731                         /* interface instanceof code */
2732
2733                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2734                                 if (super != NULL) {
2735                                         M_TEST(s1);
2736                                         emit_label_beq(cd, BRANCH_LABEL_3);
2737                                 }
2738
2739                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2740
2741                                 if (super == NULL) {
2742                                         codegen_add_patch_ref(cd, PATCHER_instanceof_interface,
2743                                                                                   iptr->sx.s23.s3.c.ref, 0);
2744                                 }
2745
2746                                 M_ILD32(REG_ITMP3,
2747                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2748                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2749
2750                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2751
2752                                 M_BLE(a);
2753                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2754                                                 OFFSET(vftbl_t, interfacetable[0]) -
2755                                                 superindex * sizeof(methodptr*));
2756                                 M_TEST(REG_ITMP1);
2757                                 M_SETNE(d);
2758
2759                                 if (super == NULL)
2760                                         emit_label_br(cd, BRANCH_LABEL_4);
2761                                 else
2762                                         emit_label(cd, BRANCH_LABEL_3);
2763                         }
2764
2765                         /* class instanceof code */
2766
2767                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2768                                 if (super == NULL) {
2769                                         emit_label(cd, BRANCH_LABEL_2);
2770                                 }
2771                                 else {
2772                                         M_TEST(s1);
2773                                         emit_label_beq(cd, BRANCH_LABEL_5);
2774                                 }
2775
2776                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2777
2778                                 if (super == NULL) {
2779                                         codegen_add_patch_ref(cd, PATCHER_instanceof_class,
2780                                                                                   iptr->sx.s23.s3.c.ref, 0);
2781                                 }
2782
2783                                 M_MOV_IMM(supervftbl, REG_ITMP2);
2784
2785                                 CODEGEN_CRITICAL_SECTION_START;
2786
2787                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
2788                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2789                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2790
2791                                 CODEGEN_CRITICAL_SECTION_END;
2792
2793                                 M_ISUB(REG_ITMP2, REG_ITMP1);
2794                                 M_CLR(d); /* may be REG_ITMP2 */
2795                                 M_ICMP(REG_ITMP3, REG_ITMP1);
2796                                 M_SETULE(d);
2797
2798                                 if (super != NULL)
2799                                         emit_label(cd, BRANCH_LABEL_5);
2800                         }
2801
2802                         if (super == NULL) {
2803                                 emit_label(cd, BRANCH_LABEL_1);
2804                                 emit_label(cd, BRANCH_LABEL_4);
2805                         }
2806
2807                         emit_store_dst(jd, iptr, d);
2808                         }
2809                         break;
2810
2811                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2812
2813                         /* check for negative sizes and copy sizes to stack if necessary  */
2814
2815                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2816
2817                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2818
2819                                 /* copy SAVEDVAR sizes to stack */
2820                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2821
2822                                 /* Already Preallocated? */
2823                                 if (!(var->flags & PREALLOC)) {
2824                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2825                                         M_LST(s2, REG_SP, s1 * 8);
2826                                 }
2827                         }
2828
2829                         /* is a patcher function set? */
2830
2831                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2832                                 codegen_add_patch_ref(cd, PATCHER_builtin_multianewarray,
2833                                                                           iptr->sx.s23.s3.c.ref, 0);
2834                         }
2835
2836                         /* a0 = dimension count */
2837
2838                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2839
2840                         /* a1 = classinfo */
2841
2842                         M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2843
2844                         /* a2 = pointer to dimensions = stack pointer */
2845
2846                         M_MOV(REG_SP, REG_A2);
2847
2848                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2849                         M_CALL(REG_ITMP1);
2850
2851                         /* check for exception before result assignment */
2852
2853                         emit_exception_check(cd, iptr);
2854
2855                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2856                         M_INTMOVE(REG_RESULT, s1);
2857                         emit_store_dst(jd, iptr, s1);
2858                         break;
2859
2860                 default:
2861                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2862                                                                                    iptr->opc);
2863                         return false;
2864         } /* switch */
2865
2866         } /* for instruction */
2867
2868         MCODECHECK(512); /* XXX require a lower number? */
2869
2870         /* At the end of a basic block we may have to append some nops,
2871            because the patcher stub calling code might be longer than the
2872            actual instruction. So codepatching does not change the
2873            following block unintentionally. */
2874
2875         if (cd->mcodeptr < cd->lastmcodeptr) {
2876                 while (cd->mcodeptr < cd->lastmcodeptr) {
2877                         M_NOP;
2878                 }
2879         }
2880
2881         } /* if (bptr -> flags >= BBREACHED) */
2882         } /* for basic block */
2883
2884         dseg_createlinenumbertable(cd);
2885
2886         /* generate stubs */
2887
2888         emit_patcher_stubs(jd);
2889
2890         /* everything's ok */
2891
2892         return true;
2893 }
2894
2895
2896 /* codegen_emit_stub_compiler **************************************************
2897
2898    Emit a stub routine which calls the compiler.
2899
2900 *******************************************************************************/
2901
2902 void codegen_emit_stub_compiler(jitdata *jd)
2903 {
2904         methodinfo  *m;
2905         codegendata *cd;
2906
2907         /* get required compiler data */
2908
2909         m  = jd->m;
2910         cd = jd->cd;
2911
2912         /* code for the stub */
2913
2914         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
2915         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
2916         M_JMP(REG_ITMP3);
2917 }
2918
2919
2920 /* codegen_emit_stub_native ****************************************************
2921
2922    Emits a stub routine which calls a native method.
2923
2924 *******************************************************************************/
2925
2926 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f)
2927 {
2928         methodinfo  *m;
2929         codeinfo    *code;
2930         codegendata *cd;
2931         methoddesc  *md;
2932         s4           nativeparams;
2933         s4           i, j;
2934         s4           t;
2935         s4           s1, s2;
2936
2937         /* get required compiler data */
2938
2939         m    = jd->m;
2940         code = jd->code;
2941         cd   = jd->cd;
2942
2943         /* initialize variables */
2944
2945         md = m->parseddesc;
2946         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
2947
2948         /* calculate stack frame size */
2949
2950         cd->stackframesize =
2951                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
2952                 sizeof(localref_table) / SIZEOF_VOID_P +
2953                 md->paramcount +
2954                 1 +                       /* functionptr, TODO: store in data segment */
2955                 nmd->memuse;
2956
2957         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
2958
2959         /* create method header */
2960
2961         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2962         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2963         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
2964         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2965         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2966         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2967         (void) dseg_addlinenumbertablesize(cd);
2968         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
2969
2970 #if defined(ENABLE_PROFILING)
2971         /* generate native method profiling code */
2972
2973         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
2974                 /* count frequency */
2975
2976                 M_MOV_IMM(code, REG_ITMP3);
2977                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
2978         }
2979 #endif
2980
2981         /* generate stub code */
2982
2983         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2984
2985 #if !defined(NDEBUG)
2986         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2987                 emit_verbosecall_enter(jd);
2988 #endif
2989
2990         /* get function address (this must happen before the stackframeinfo) */
2991
2992 #if !defined(WITH_STATIC_CLASSPATH)
2993         if (f == NULL)
2994                 codegen_add_patch_ref(cd, PATCHER_resolve_native, m, 0);
2995 #endif
2996
2997         M_MOV_IMM(f, REG_ITMP3);
2998
2999         /* save integer and float argument registers */
3000
3001         for (i = 0; i < md->paramcount; i++) {
3002                 if (!md->params[i].inmemory) {
3003                         s1 = md->params[i].regoff;
3004
3005                         switch (md->paramtypes[i].type) {
3006                         case TYPE_INT:
3007                         case TYPE_LNG:
3008                         case TYPE_ADR:
3009                                 M_LST(s1, REG_SP, i * 8);
3010                                 break;
3011                         case TYPE_FLT:
3012                         case TYPE_DBL:
3013                                 M_DST(s1, REG_SP, i * 8);
3014                                 break;
3015                         }
3016                 }
3017         }
3018
3019         M_AST(REG_ITMP3, REG_SP, md->paramcount * 8);
3020
3021         /* create dynamic stack info */
3022
3023         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3024         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3025         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3026         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3027         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3028         M_CALL(REG_ITMP1);
3029
3030         /* restore integer and float argument registers */
3031
3032         for (i = 0; i < md->paramcount; i++) {
3033                 if (!md->params[i].inmemory) {
3034                         s1 = md->params[i].regoff;
3035
3036                         switch (md->paramtypes[i].type) {
3037                         case TYPE_INT:
3038                         case TYPE_LNG:
3039                         case TYPE_ADR:
3040                                 M_LLD(s1, REG_SP, i * 8);
3041                                 break;
3042                         case TYPE_FLT:
3043                         case TYPE_DBL:
3044                                 M_DLD(s1, REG_SP, i * 8);
3045                                 break;
3046                         }
3047                 }
3048         }
3049
3050         M_ALD(REG_ITMP3, REG_SP, md->paramcount * 8);
3051
3052         /* copy or spill arguments to new locations */
3053
3054         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3055                 t  = md->paramtypes[i].type;
3056                 s2 = nmd->params[j].regoff;
3057
3058                 if (IS_INT_LNG_TYPE(t)) {
3059                         if (!md->params[i].inmemory) {
3060                                 s1 = md->params[i].regoff;
3061
3062                                 if (!nmd->params[j].inmemory)
3063                                         M_INTMOVE(s1, s2);
3064                                 else
3065                                         M_LST(s1, REG_SP, s2);
3066                         }
3067                         else {
3068                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3069                                 M_LLD(REG_ITMP1, REG_SP, s1);
3070                                 M_LST(REG_ITMP1, REG_SP, s2);
3071                         }
3072                 }
3073                 else {
3074                         /* We only copy spilled float arguments, as the float
3075                            argument registers keep unchanged. */
3076
3077                         if (md->params[i].inmemory) {
3078                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3079
3080                                 if (IS_2_WORD_TYPE(t)) {
3081                                         M_DLD(REG_FTMP1, REG_SP, s1);
3082                                         M_DST(REG_FTMP1, REG_SP, s2);
3083                                 }
3084                                 else {
3085                                         M_FLD(REG_FTMP1, REG_SP, s1);
3086                                         M_FST(REG_FTMP1, REG_SP, s2);
3087                                 }
3088                         }
3089                 }
3090         }
3091
3092         /* put class into second argument register */
3093
3094         if (m->flags & ACC_STATIC)
3095                 M_MOV_IMM(m->class, REG_A1);
3096
3097         /* put env into first argument register */
3098
3099         M_MOV_IMM(_Jv_env, REG_A0);
3100
3101         /* do the native function call */
3102
3103         M_CALL(REG_ITMP3);
3104
3105         /* save return value */
3106
3107         switch (md->returntype.type) {
3108         case TYPE_INT:
3109         case TYPE_LNG:
3110         case TYPE_ADR:
3111                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3112                 break;
3113         case TYPE_FLT:
3114         case TYPE_DBL:
3115                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3116                 break;
3117         case TYPE_VOID:
3118                 break;
3119         }
3120
3121 #if !defined(NDEBUG)
3122         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3123                 emit_verbosecall_exit(jd);
3124 #endif
3125
3126         /* remove native stackframe info */
3127
3128         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3129         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3130         M_CALL(REG_ITMP1);
3131         M_MOV(REG_RESULT, REG_ITMP3);
3132
3133         /* restore return value */
3134
3135         switch (md->returntype.type) {
3136         case TYPE_INT:
3137         case TYPE_LNG:
3138         case TYPE_ADR:
3139                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3140                 break;
3141         case TYPE_FLT:
3142         case TYPE_DBL:
3143                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3144                 break;
3145         case TYPE_VOID:
3146                 break;
3147         }
3148
3149         /* remove stackframe */
3150
3151         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3152
3153         /* test for exception */
3154
3155         M_TEST(REG_ITMP3);
3156         M_BNE(1);
3157         M_RET;
3158
3159         /* handle exception */
3160
3161         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3162         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3163         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3164
3165         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3166         M_JMP(REG_ITMP3);
3167
3168         /* generate patcher stubs */
3169
3170         emit_patcher_stubs(jd);
3171 }
3172
3173
3174 /*
3175  * These are local overrides for various environment variables in Emacs.
3176  * Please do not remove this and leave it at the end of the file, where
3177  * Emacs will automagically detect them.
3178  * ---------------------------------------------------------------------
3179  * Local variables:
3180  * mode: c
3181  * indent-tabs-mode: t
3182  * c-basic-offset: 4
3183  * tab-width: 4
3184  * End:
3185  * vim:noexpandtab:sw=4:ts=4:
3186  */