src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    $Id: codegen.c 7754 2007-04-17 23:18:15Z twisti $
  26
  27 */
  28
  29
  30 #include "config.h"
  31
  32 #include <assert.h>
  33 #include <stdio.h>
  34
  35 #include "vm/types.h"
  36
  37 #include "md-abi.h"
  38
  39 #include "vm/jit/x86_64/arch.h"
  40 #include "vm/jit/x86_64/codegen.h"
  41 #include "vm/jit/x86_64/emit.h"
  42
  43 #include "mm/memory.h"
  44 #include "native/jni.h"
  45 #include "native/native.h"
  46
  47 #if defined(ENABLE_THREADS)
  48 # include "threads/native/lock.h"
  49 #endif
  50
  51 #include "vm/builtin.h"
  52 #include "vm/exceptions.h"
  53 #include "vm/global.h"
  54 #include "vm/stringlocal.h"
  55 #include "vm/vm.h"
  56
  57 #include "vm/jit/abi.h"
  58 #include "vm/jit/asmpart.h"
  59 #include "vm/jit/codegen-common.h"
  60 #include "vm/jit/dseg.h"
  61 #include "vm/jit/emit-common.h"
  62 #include "vm/jit/jit.h"
  63 #include "vm/jit/methodheader.h"
  64 #include "vm/jit/parse.h"
  65 #include "vm/jit/patcher.h"
  66 #include "vm/jit/reg.h"
  67 #include "vm/jit/replace.h"
  68 #include "vm/jit/stacktrace.h"
  69
  70 #if defined(ENABLE_LSRA)
  71 # include "vm/jit/allocator/lsra.h"
  72 #endif
  73
  74 #include "vmcore/loader.h"
  75 #include "vmcore/options.h"
  76 #include "vmcore/statistics.h"
  77
  78
  79 /* codegen_emit ****************************************************************
  80
  81    Generates machine code.
  82
  83 *******************************************************************************/
  84
  85 bool codegen_emit(jitdata *jd)
  86 {
  87         methodinfo         *m;
  88         codeinfo           *code;
  89         codegendata        *cd;
  90         registerdata       *rd;
  91         s4                  len, s1, s2, s3, d, disp;
  92         u2                  currentline;
  93         ptrint              a;
  94         varinfo            *var, *dst;
  95         basicblock         *bptr;
  96         instruction        *iptr;
  97         exception_entry    *ex;
  98         constant_classref  *cr;
  99         unresolved_class   *uc;
 100         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 101         unresolved_method  *um;
 102         builtintable_entry *bte;
 103         methoddesc         *md;
 104         fieldinfo          *fi;
 105         unresolved_field   *uf;
 106         s4                  fieldtype;
 107         s4                 varindex;
 108
 109         /* get required compiler data */
 110
 111         m    = jd->m;
 112         code = jd->code;
 113         cd   = jd->cd;
 114         rd   = jd->rd;
 115
 116         /* prevent compiler warnings */
 117
 118         d   = 0;
 119         lm  = NULL;
 120         um  = NULL;
 121         bte = NULL;
 122
 123         {
 124         s4 i, p, t, l;
 125         s4 savedregs_num;
 126
 127         savedregs_num = 0;
 128
 129         /* space to save used callee saved registers */
 130
 131         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 132         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 133
 134         cd->stackframesize = rd->memuse + savedregs_num;
 135
 136 #if defined(ENABLE_THREADS)
 137         /* space to save argument of monitor_enter */
 138
 139         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 140                 cd->stackframesize++;
 141 #endif
 142
 143         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 144            native code e.g. libc or jni (alignment problems with
 145            movaps). */
 146
 147         if (!jd->isleafmethod || opt_verbosecall)
 148                 cd->stackframesize |= 0x1;
 149
 150         /* create method header */
 151
 152         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 153         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 154
 155 #if defined(ENABLE_THREADS)
 156         /* IsSync contains the offset relative to the stack pointer for the
 157            argument of monitor_exit used in the exception handler. Since the
 158            offset could be zero and give a wrong meaning of the flag it is
 159            offset by one.
 160         */
 161
 162         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 163                 (void) dseg_add_unique_s4(cd, (rd->memuse + 1) * 8); /* IsSync        */
 164         else
 165 #endif
 166                 (void) dseg_add_unique_s4(cd, 0);                  /* IsSync          */
 167
 168         (void) dseg_add_unique_s4(cd, jd->isleafmethod);               /* IsLeaf  */
 169         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 170         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 171
 172         (void) dseg_addlinenumbertablesize(cd);
 173
 174         (void) dseg_add_unique_s4(cd, jd->exceptiontablelength); /* ExTableSize   */
 175
 176         /* create exception table */
 177
 178         for (ex = jd->exceptiontable; ex != NULL; ex = ex->down) {
 179                 dseg_add_target(cd, ex->start);
 180                 dseg_add_target(cd, ex->end);
 181                 dseg_add_target(cd, ex->handler);
 182                 (void) dseg_add_unique_address(cd, ex->catchtype.any);
 183         }
 184
 185 #if defined(ENABLE_PROFILING)
 186         /* generate method profiling code */
 187
 188         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 189                 /* count frequency */
 190
 191                 M_MOV_IMM(code, REG_ITMP3);
 192                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 193
 194                 PROFILE_CYCLE_START;
 195         }
 196 #endif
 197
 198         /* create stack frame (if necessary) */
 199
 200         if (cd->stackframesize)
 201                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 202
 203         /* save used callee saved registers */
 204
 205         p = cd->stackframesize;
 206         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 207                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 208         }
 209         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 210                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 211         }
 212
 213         /* take arguments out of register or stack frame */
 214
 215         md = m->parseddesc;
 216
 217         for (p = 0, l = 0; p < md->paramcount; p++) {
 218                 t = md->paramtypes[p].type;
 219
 220                 varindex = jd->local_map[l * 5 + t];
 221
 222                 l++;
 223                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 224                         l++;
 225
 226                 if (varindex == UNUSED)
 227                         continue;
 228
 229                 var = VAR(varindex);
 230
 231                 s1 = md->params[p].regoff;
 232
 233                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 234                         if (!md->params[p].inmemory) {           /* register arguments    */
 235                                 if (!IS_INMEMORY(var->flags))
 236                                         M_INTMOVE(s1, var->vv.regoff);
 237                                 else
 238                                     M_LST(s1, REG_SP, var->vv.regoff * 8);
 239                         }
 240                         else {                                 /* stack arguments       */
 241                                 if (!IS_INMEMORY(var->flags))
 242                                         /* + 8 for return address */
 243                                         M_LLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 244                                 else
 245                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 246                         }
 247                 }
 248                 else {                                     /* floating args         */
 249                         if (!md->params[p].inmemory) {           /* register arguments    */
 250                                 if (!IS_INMEMORY(var->flags))
 251                                         M_FLTMOVE(s1, var->vv.regoff);
 252                                 else
 253                                         M_DST(s1, REG_SP, var->vv.regoff * 8);
 254                         }
 255                         else {                                 /* stack arguments       */
 256                                 if (!IS_INMEMORY(var->flags))
 257                                         M_DLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 258                                 else
 259                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 260                         }
 261                 }
 262         }
 263
 264         /* save monitorenter argument */
 265
 266 #if defined(ENABLE_THREADS)
 267         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 268                 /* stack offset for monitor argument */
 269
 270                 s1 = rd->memuse;
 271
 272                 if (opt_verbosecall) {
 273                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 274
 275                         for (p = 0; p < INT_ARG_CNT; p++)
 276                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 277
 278                         for (p = 0; p < FLT_ARG_CNT; p++)
 279                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 280
 281                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 282                 }
 283
 284                 /* decide which monitor enter function to call */
 285
 286                 if (m->flags & ACC_STATIC) {
 287                         M_MOV_IMM(&m->class->object.header, REG_A0);
 288                 }
 289                 else {
 290                         M_TEST(REG_A0);
 291                         M_BNE(8);
 292                         M_ALD_MEM(REG_A0, EXCEPTION_HARDWARE_NULLPOINTER);
 293                 }
 294
 295                 M_AST(REG_A0, REG_SP, s1 * 8);
 296                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 297                 M_CALL(REG_ITMP1);
 298
 299                 if (opt_verbosecall) {
 300                         for (p = 0; p < INT_ARG_CNT; p++)
 301                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 302
 303                         for (p = 0; p < FLT_ARG_CNT; p++)
 304                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 305
 306                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 307                 }
 308         }
 309 #endif
 310
 311 #if !defined(NDEBUG)
 312         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 313                 emit_verbosecall_enter(jd);
 314 #endif /* !defined(NDEBUG) */
 315
 316         }
 317
 318         /* end of header generation */
 319
 320         /* create replacement points */
 321
 322         REPLACEMENT_POINTS_INIT(cd, jd);
 323
 324         /* walk through all basic blocks */
 325
 326         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 327
 328                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 329
 330                 if (bptr->flags >= BBREACHED) {
 331
 332                 /* branch resolving */
 333
 334                 codegen_resolve_branchrefs(cd, bptr);
 335
 336                 /* handle replacement points */
 337
 338                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 339
 340                 /* copy interface registers to their destination */
 341
 342                 len = bptr->indepth;
 343                 MCODECHECK(512);
 344
 345 #if defined(ENABLE_PROFILING)
 346                 /* generate basicblock profiling code */
 347
 348                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 349                         /* count frequency */
 350
 351                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 352                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 353
 354                         /* if this is an exception handler, start profiling again */
 355
 356                         if (bptr->type == BBTYPE_EXH)
 357                                 PROFILE_CYCLE_START;
 358                 }
 359 #endif
 360
 361 #if defined(ENABLE_LSRA)
 362                 if (opt_lsra) {
 363                         while (len) {
 364                                 len--;
 365                                 src = bptr->invars[len];
 366                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 367                                         if (bptr->type == BBTYPE_EXH) {
 368 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 369                                                 if (!IS_INMEMORY(src->flags))
 370                                                         d= src->vv.regoff;
 371                                                 else
 372                                                         d=REG_ITMP1;
 373                                                 M_INTMOVE(REG_ITMP1, d);
 374                                                 emit_store(jd, NULL, src, d);
 375                                         }
 376                                 }
 377                         }
 378
 379                 } else {
 380 #endif
 381
 382                 while (len) {
 383                         len--;
 384                         var = VAR(bptr->invars[len]);
 385                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 386                                 if (bptr->type == BBTYPE_EXH) {
 387                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 388                                         M_INTMOVE(REG_ITMP1, d);
 389                                         emit_store(jd, NULL, var, d);
 390                                 }
 391                         }
 392                         else {
 393                                 assert((var->flags & INOUT));
 394                         }
 395                 }
 396 #if defined(ENABLE_LSRA)
 397                 }
 398 #endif
 399                 /* walk through all instructions */
 400
 401                 len = bptr->icount;
 402                 currentline = 0;
 403
 404                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 405                         if (iptr->line != currentline) {
 406                                 dseg_addlinenumber(cd, iptr->line);
 407                                 currentline = iptr->line;
 408                         }
 409
 410                         MCODECHECK(1024);                         /* 1KB should be enough */
 411
 412                 switch (iptr->opc) {
 413                 case ICMD_NOP:        /* ...  ==> ...                                 */
 414                 case ICMD_POP:        /* ..., value  ==> ...                          */
 415                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 416                         break;
 417
 418                 case ICMD_INLINE_START:
 419
 420                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 421                         break;
 422
 423                 case ICMD_INLINE_BODY:
 424
 425                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 426                         dseg_addlinenumber_inline_start(cd, iptr);
 427                         dseg_addlinenumber(cd, iptr->line);
 428                         break;
 429
 430                 case ICMD_INLINE_END:
 431
 432                         dseg_addlinenumber_inline_end(cd, iptr);
 433                         dseg_addlinenumber(cd, iptr->line);
 434                         break;
 435
 436                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 437
 438                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 439                         emit_nullpointer_check(cd, iptr, s1);
 440                         break;
 441
 442                 /* constant operations ************************************************/
 443
 444                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 445
 446                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 447                         ICONST(d, iptr->sx.val.i);
 448                         emit_store_dst(jd, iptr, d);
 449                         break;
 450
 451                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 452
 453                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 454                         LCONST(d, iptr->sx.val.l);
 455                         emit_store_dst(jd, iptr, d);
 456                         break;
 457
 458                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 459
 460                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 461                         disp = dseg_add_float(cd, iptr->sx.val.f);
 462                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 463                         emit_store_dst(jd, iptr, d);
 464                         break;
 465
 466                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 467
 468                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 469                         disp = dseg_add_double(cd, iptr->sx.val.d);
 470                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 471                         emit_store_dst(jd, iptr, d);
 472                         break;
 473
 474                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 475
 476                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 477
 478                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 479                                 cr = iptr->sx.val.c.ref;
 480
 481 /*                              PROFILE_CYCLE_STOP; */
 482
 483                                 codegen_add_patch_ref(cd, PATCHER_aconst, cr, 0);
 484
 485 /*                              PROFILE_CYCLE_START; */
 486
 487                                 M_MOV_IMM(NULL, d);
 488
 489                         } else {
 490                                 if (iptr->sx.val.anyptr == 0)
 491                                         M_CLR(d);
 492                                 else
 493                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 494                         }
 495                         emit_store_dst(jd, iptr, d);
 496                         break;
 497
 498
 499                 /* load/store/copy/move operations ************************************/
 500
 501                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 502                 case ICMD_ALOAD:      /* s1 = local variable                          */
 503                 case ICMD_LLOAD:
 504                 case ICMD_FLOAD:
 505                 case ICMD_DLOAD:
 506                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 507                 case ICMD_LSTORE:
 508                 case ICMD_FSTORE:
 509                 case ICMD_DSTORE:
 510                 case ICMD_COPY:
 511                 case ICMD_MOVE:
 512
 513                         emit_copy(jd, iptr);
 514                         break;
 515
 516                 case ICMD_ASTORE:
 517                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 518                                 emit_copy(jd, iptr);
 519                         break;
 520
 521                 /* integer operations *************************************************/
 522
 523                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 524
 525                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 526                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 527                         M_INTMOVE(s1, d);
 528                         M_INEG(d);
 529                         emit_store_dst(jd, iptr, d);
 530                         break;
 531
 532                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 533
 534                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 535                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 536                         M_INTMOVE(s1, d);
 537                         M_LNEG(d);
 538                         emit_store_dst(jd, iptr, d);
 539                         break;
 540
 541                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 542
 543                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 544                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 545                         M_ISEXT(s1, d);
 546                         emit_store_dst(jd, iptr, d);
 547                         break;
 548
 549                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 550
 551                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 552                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 553                         M_IMOV(s1, d);
 554                         emit_store_dst(jd, iptr, d);
 555                         break;
 556
 557                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 558
 559                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 560                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 561                         M_BSEXT(s1, d);
 562                         emit_store_dst(jd, iptr, d);
 563                         break;
 564
 565                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 566
 567                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 568                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 569                         M_CZEXT(s1, d);
 570                         emit_store_dst(jd, iptr, d);
 571                         break;
 572
 573                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 574
 575                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 576                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 577                         M_SSEXT(s1, d);
 578                         emit_store_dst(jd, iptr, d);
 579                         break;
 580
 581
 582                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 583
 584                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 585                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 586                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 587                         if (s2 == d)
 588                                 M_IADD(s1, d);
 589                         else {
 590                                 M_INTMOVE(s1, d);
 591                                 M_IADD(s2, d);
 592                         }
 593                         emit_store_dst(jd, iptr, d);
 594                         break;
 595
 596                 case ICMD_IINC:
 597                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 598                                       /* sx.val.i = constant                             */
 599
 600                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 601                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 602
 603                         /* Using inc and dec is not faster than add (tested with
 604                            sieve). */
 605
 606                         M_INTMOVE(s1, d);
 607                         M_IADD_IMM(iptr->sx.val.i, d);
 608                         emit_store_dst(jd, iptr, d);
 609                         break;
 610
 611                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 612
 613                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 614                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 615                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 616                         if (s2 == d)
 617                                 M_LADD(s1, d);
 618                         else {
 619                                 M_INTMOVE(s1, d);
 620                                 M_LADD(s2, d);
 621                         }
 622                         emit_store_dst(jd, iptr, d);
 623                         break;
 624
 625                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 626                                       /* sx.val.l = constant                             */
 627
 628                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 629                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 630                         M_INTMOVE(s1, d);
 631                         if (IS_IMM32(iptr->sx.val.l))
 632                                 M_LADD_IMM(iptr->sx.val.l, d);
 633                         else {
 634                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 635                                 M_LADD(REG_ITMP2, d);
 636                         }
 637                         emit_store_dst(jd, iptr, d);
 638                         break;
 639
 640                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 641
 642                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 643                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 644                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 645                         if (s2 == d) {
 646                                 M_INTMOVE(s1, REG_ITMP1);
 647                                 M_ISUB(s2, REG_ITMP1);
 648                                 M_INTMOVE(REG_ITMP1, d);
 649                         } else {
 650                                 M_INTMOVE(s1, d);
 651                                 M_ISUB(s2, d);
 652                         }
 653                         emit_store_dst(jd, iptr, d);
 654                         break;
 655
 656                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 657                                       /* sx.val.i = constant                             */
 658
 659                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 660                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 661                         M_INTMOVE(s1, d);
 662                         M_ISUB_IMM(iptr->sx.val.i, d);
 663                         emit_store_dst(jd, iptr, d);
 664                         break;
 665
 666                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 667
 668                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 669                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 670                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 671                         if (s2 == d) {
 672                                 M_INTMOVE(s1, REG_ITMP1);
 673                                 M_LSUB(s2, REG_ITMP1);
 674                                 M_INTMOVE(REG_ITMP1, d);
 675                         } else {
 676                                 M_INTMOVE(s1, d);
 677                                 M_LSUB(s2, d);
 678                         }
 679                         emit_store_dst(jd, iptr, d);
 680                         break;
 681
 682                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 683                                       /* sx.val.l = constant                             */
 684
 685                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 686                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 687                         M_INTMOVE(s1, d);
 688                         if (IS_IMM32(iptr->sx.val.l))
 689                                 M_LSUB_IMM(iptr->sx.val.l, d);
 690                         else {
 691                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 692                                 M_LSUB(REG_ITMP2, d);
 693                         }
 694                         emit_store_dst(jd, iptr, d);
 695                         break;
 696
 697                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 698
 699                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 700                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 701                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 702                         if (s2 == d)
 703                                 M_IMUL(s1, d);
 704                         else {
 705                                 M_INTMOVE(s1, d);
 706                                 M_IMUL(s2, d);
 707                         }
 708                         emit_store_dst(jd, iptr, d);
 709                         break;
 710
 711                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 712                                       /* sx.val.i = constant                             */
 713
 714                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 715                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 716                         if (iptr->sx.val.i == 2) {
 717                                 M_INTMOVE(s1, d);
 718                                 M_ISLL_IMM(1, d);
 719                         } else
 720                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 721                         emit_store_dst(jd, iptr, d);
 722                         break;
 723
 724                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 725
 726                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 727                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 728                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 729                         if (s2 == d)
 730                                 M_LMUL(s1, d);
 731                         else {
 732                                 M_INTMOVE(s1, d);
 733                                 M_LMUL(s2, d);
 734                         }
 735                         emit_store_dst(jd, iptr, d);
 736                         break;
 737
 738                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 739                                       /* sx.val.l = constant                             */
 740
 741                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 742                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 743                         if (IS_IMM32(iptr->sx.val.l))
 744                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 745                         else {
 746                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 747                                 M_INTMOVE(s1, d);
 748                                 M_LMUL(REG_ITMP2, d);
 749                         }
 750                         emit_store_dst(jd, iptr, d);
 751                         break;
 752
 753                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 754
 755                         s1 = emit_load_s1(jd, iptr, RAX);
 756                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 757                         d = codegen_reg_of_dst(jd, iptr, RAX);
 758
 759                         M_INTMOVE(s1, RAX);
 760                         M_INTMOVE(s2, REG_ITMP3);
 761                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 762
 763                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 764
 765                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 766                         M_BNE(4 + 6);
 767                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 768                         M_BEQ(1 + 3);                                          /* 6 bytes */
 769
 770                         emit_cltd(cd);                                         /* 1 byte  */
 771                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 772
 773                         M_INTMOVE(RAX, d);
 774                         emit_store_dst(jd, iptr, d);
 775                         dst = VAROP(iptr->dst);
 776                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 777                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 778                         break;
 779
 780                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 781
 782                         s1 = emit_load_s1(jd, iptr, RAX);
 783                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 784                         d = codegen_reg_of_dst(jd, iptr, RDX);
 785
 786                         M_INTMOVE(s1, RAX);
 787                         M_INTMOVE(s2, REG_ITMP3);
 788                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 789
 790                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 791
 792                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 793                         M_BNE(3 + 4 + 6);
 794                         M_CLR(RDX);                                            /* 3 bytes */
 795                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 796                         M_BEQ(1 + 3);                                          /* 6 bytes */
 797
 798                         emit_cltd(cd);                                         /* 1 byte  */
 799                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 800
 801                         M_INTMOVE(RDX, d);
 802                         emit_store_dst(jd, iptr, d);
 803                         dst = VAROP(iptr->dst);
 804                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 805                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 806                         break;
 807
 808                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 809                                       /* sx.val.i = constant                             */
 810
 811                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 812                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 813                         M_INTMOVE(s1, REG_ITMP1);
 814                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 815                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 816                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 817                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 818                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 819                         emit_store_dst(jd, iptr, d);
 820                         break;
 821
 822                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 823                                       /* sx.val.i = constant                             */
 824
 825                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 826                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 827                         M_INTMOVE(s1, REG_ITMP1);
 828                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 829                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 830                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 831                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 832                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 833                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 834                         emit_store_dst(jd, iptr, d);
 835                         break;
 836
 837
 838                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 839
 840                         s1 = emit_load_s1(jd, iptr, RAX);
 841                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 842                         d = codegen_reg_of_dst(jd, iptr, RAX);
 843
 844                         M_INTMOVE(s1, RAX);
 845                         M_INTMOVE(s2, REG_ITMP3);
 846                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 847
 848                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 849
 850                         /* check as described in jvm spec */
 851                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 852                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 853                         M_BNE(4 + 6);
 854                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 855                         M_BEQ(2 + 3);                                          /* 6 bytes */
 856
 857                         emit_cqto(cd);                                         /* 2 bytes */
 858                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 859
 860                         M_INTMOVE(RAX, d);
 861                         emit_store_dst(jd, iptr, d);
 862                         dst = VAROP(iptr->dst);
 863                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 864                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 865                         break;
 866
 867                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 868
 869                         s1 = emit_load_s1(jd, iptr, RAX);
 870                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 871                         d = codegen_reg_of_dst(jd, iptr, RDX);
 872
 873                         M_INTMOVE(s1, RAX);
 874                         M_INTMOVE(s2, REG_ITMP3);
 875                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 876
 877                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 878
 879                         /* check as described in jvm spec */
 880                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 881                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 882                         M_BNE(3 + 4 + 6);
 883                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 884                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 885                         M_BEQ(2 + 3);                                          /* 6 bytes */
 886
 887                         emit_cqto(cd);                                         /* 2 bytes */
 888                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 889
 890                         M_INTMOVE(RDX, d);
 891                         emit_store_dst(jd, iptr, d);
 892                         dst = VAROP(iptr->dst);
 893                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 894                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 895                         break;
 896
 897                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 898                                       /* sx.val.i = constant                             */
 899
 900                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 901                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 902                         M_INTMOVE(s1, REG_ITMP1);
 903                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 904                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 905                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 906                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 907                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 908                         emit_store_dst(jd, iptr, d);
 909                         break;
 910
 911                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 912                                       /* sx.val.l = constant                             */
 913
 914                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 915                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 916                         M_INTMOVE(s1, REG_ITMP1);
 917                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 918                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 919                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 920                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 921                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 922                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 923                         emit_store_dst(jd, iptr, d);
 924                         break;
 925
 926                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 927
 928                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 929                         emit_ishift(jd, SHIFT_SHL, iptr);
 930                         break;
 931
 932                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 933                                       /* sx.val.i = constant                             */
 934
 935                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 936                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 937                         M_INTMOVE(s1, d);
 938                         M_ISLL_IMM(iptr->sx.val.i, d);
 939                         emit_store_dst(jd, iptr, d);
 940                         break;
 941
 942                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 943
 944                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 945                         emit_ishift(jd, SHIFT_SAR, iptr);
 946                         break;
 947
 948                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 949                                       /* sx.val.i = constant                             */
 950
 951                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 952                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 953                         M_INTMOVE(s1, d);
 954                         M_ISRA_IMM(iptr->sx.val.i, d);
 955                         emit_store_dst(jd, iptr, d);
 956                         break;
 957
 958                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 959
 960                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 961                         emit_ishift(jd, SHIFT_SHR, iptr);
 962                         break;
 963
 964                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 965                                       /* sx.val.i = constant                             */
 966
 967                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 968                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 969                         M_INTMOVE(s1, d);
 970                         M_ISRL_IMM(iptr->sx.val.i, d);
 971                         emit_store_dst(jd, iptr, d);
 972                         break;
 973
 974                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 975
 976                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 977                         emit_lshift(jd, SHIFT_SHL, iptr);
 978                         break;
 979
 980         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 981                                           /* sx.val.i = constant                             */
 982
 983                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 984                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 985                         M_INTMOVE(s1, d);
 986                         M_LSLL_IMM(iptr->sx.val.i, d);
 987                         emit_store_dst(jd, iptr, d);
 988                         break;
 989
 990                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 991
 992                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 993                         emit_lshift(jd, SHIFT_SAR, iptr);
 994                         break;
 995
 996                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 997                                       /* sx.val.i = constant                             */
 998
 999                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1000                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1001                         M_INTMOVE(s1, d);
1002                         M_LSRA_IMM(iptr->sx.val.i, d);
1003                         emit_store_dst(jd, iptr, d);
1004                         break;
1005
1006                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1007
1008                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1009                         emit_lshift(jd, SHIFT_SHR, iptr);
1010                         break;
1011
1012                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1013                                       /* sx.val.l = constant                             */
1014
1015                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1016                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1017                         M_INTMOVE(s1, d);
1018                         M_LSRL_IMM(iptr->sx.val.i, d);
1019                         emit_store_dst(jd, iptr, d);
1020                         break;
1021
1022                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1023
1024                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1025                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1026                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1027                         if (s2 == d)
1028                                 M_IAND(s1, d);
1029                         else {
1030                                 M_INTMOVE(s1, d);
1031                                 M_IAND(s2, d);
1032                         }
1033                         emit_store_dst(jd, iptr, d);
1034                         break;
1035
1036                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1037                                       /* sx.val.i = constant                             */
1038
1039                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1040                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1041                         M_INTMOVE(s1, d);
1042                         M_IAND_IMM(iptr->sx.val.i, d);
1043                         emit_store_dst(jd, iptr, d);
1044                         break;
1045
1046                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1047
1048                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1049                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1050                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1051                         if (s2 == d)
1052                                 M_LAND(s1, d);
1053                         else {
1054                                 M_INTMOVE(s1, d);
1055                                 M_LAND(s2, d);
1056                         }
1057                         emit_store_dst(jd, iptr, d);
1058                         break;
1059
1060                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1061                                       /* sx.val.l = constant                             */
1062
1063                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1064                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1065                         M_INTMOVE(s1, d);
1066                         if (IS_IMM32(iptr->sx.val.l))
1067                                 M_LAND_IMM(iptr->sx.val.l, d);
1068                         else {
1069                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1070                                 M_LAND(REG_ITMP2, d);
1071                         }
1072                         emit_store_dst(jd, iptr, d);
1073                         break;
1074
1075                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1076
1077                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1078                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1079                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1080                         if (s2 == d)
1081                                 M_IOR(s1, d);
1082                         else {
1083                                 M_INTMOVE(s1, d);
1084                                 M_IOR(s2, d);
1085                         }
1086                         emit_store_dst(jd, iptr, d);
1087                         break;
1088
1089                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1090                                       /* sx.val.i = constant                             */
1091
1092                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1093                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1094                         M_INTMOVE(s1, d);
1095                         M_IOR_IMM(iptr->sx.val.i, d);
1096                         emit_store_dst(jd, iptr, d);
1097                         break;
1098
1099                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1100
1101                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1102                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1103                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1104                         if (s2 == d)
1105                                 M_LOR(s1, d);
1106                         else {
1107                                 M_INTMOVE(s1, d);
1108                                 M_LOR(s2, d);
1109                         }
1110                         emit_store_dst(jd, iptr, d);
1111                         break;
1112
1113                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1114                                       /* sx.val.l = constant                             */
1115
1116                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1117                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1118                         M_INTMOVE(s1, d);
1119                         if (IS_IMM32(iptr->sx.val.l))
1120                                 M_LOR_IMM(iptr->sx.val.l, d);
1121                         else {
1122                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1123                                 M_LOR(REG_ITMP2, d);
1124                         }
1125                         emit_store_dst(jd, iptr, d);
1126                         break;
1127
1128                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1129
1130                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1131                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1132                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1133                         if (s2 == d)
1134                                 M_IXOR(s1, d);
1135                         else {
1136                                 M_INTMOVE(s1, d);
1137                                 M_IXOR(s2, d);
1138                         }
1139                         emit_store_dst(jd, iptr, d);
1140                         break;
1141
1142                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1143                                       /* sx.val.i = constant                             */
1144
1145                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1146                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1147                         M_INTMOVE(s1, d);
1148                         M_IXOR_IMM(iptr->sx.val.i, d);
1149                         emit_store_dst(jd, iptr, d);
1150                         break;
1151
1152                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1153
1154                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1155                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1156                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1157                         if (s2 == d)
1158                                 M_LXOR(s1, d);
1159                         else {
1160                                 M_INTMOVE(s1, d);
1161                                 M_LXOR(s2, d);
1162                         }
1163                         emit_store_dst(jd, iptr, d);
1164                         break;
1165
1166                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1167                                       /* sx.val.l = constant                             */
1168
1169                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1170                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1171                         M_INTMOVE(s1, d);
1172                         if (IS_IMM32(iptr->sx.val.l))
1173                                 M_LXOR_IMM(iptr->sx.val.l, d);
1174                         else {
1175                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1176                                 M_LXOR(REG_ITMP2, d);
1177                         }
1178                         emit_store_dst(jd, iptr, d);
1179                         break;
1180
1181
1182                 /* floating operations ************************************************/
1183
1184                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1185
1186                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1187                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1188                         disp = dseg_add_s4(cd, 0x80000000);
1189                         M_FLTMOVE(s1, d);
1190                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1191                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1192                         emit_store_dst(jd, iptr, d);
1193                         break;
1194
1195                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1196
1197                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1198                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1199                         disp = dseg_add_s8(cd, 0x8000000000000000);
1200                         M_FLTMOVE(s1, d);
1201                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1202                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1203                         emit_store_dst(jd, iptr, d);
1204                         break;
1205
1206                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1207
1208                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1209                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1210                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1211                         if (s2 == d)
1212                                 M_FADD(s1, d);
1213                         else {
1214                                 M_FLTMOVE(s1, d);
1215                                 M_FADD(s2, d);
1216                         }
1217                         emit_store_dst(jd, iptr, d);
1218                         break;
1219
1220                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1221
1222                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1223                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1224                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1225                         if (s2 == d)
1226                                 M_DADD(s1, d);
1227                         else {
1228                                 M_FLTMOVE(s1, d);
1229                                 M_DADD(s2, d);
1230                         }
1231                         emit_store_dst(jd, iptr, d);
1232                         break;
1233
1234                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1235
1236                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1237                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1238                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1239                         if (s2 == d) {
1240                                 M_FLTMOVE(s2, REG_FTMP2);
1241                                 s2 = REG_FTMP2;
1242                         }
1243                         M_FLTMOVE(s1, d);
1244                         M_FSUB(s2, d);
1245                         emit_store_dst(jd, iptr, d);
1246                         break;
1247
1248                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1249
1250                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1251                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1252                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1253                         if (s2 == d) {
1254                                 M_FLTMOVE(s2, REG_FTMP2);
1255                                 s2 = REG_FTMP2;
1256                         }
1257                         M_FLTMOVE(s1, d);
1258                         M_DSUB(s2, d);
1259                         emit_store_dst(jd, iptr, d);
1260                         break;
1261
1262                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1263
1264                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1265                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1266                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1267                         if (s2 == d)
1268                                 M_FMUL(s1, d);
1269                         else {
1270                                 M_FLTMOVE(s1, d);
1271                                 M_FMUL(s2, d);
1272                         }
1273                         emit_store_dst(jd, iptr, d);
1274                         break;
1275
1276                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1277
1278                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1279                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1280                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1281                         if (s2 == d)
1282                                 M_DMUL(s1, d);
1283                         else {
1284                                 M_FLTMOVE(s1, d);
1285                                 M_DMUL(s2, d);
1286                         }
1287                         emit_store_dst(jd, iptr, d);
1288                         break;
1289
1290                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1291
1292                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1293                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1294                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1295                         if (s2 == d) {
1296                                 M_FLTMOVE(s2, REG_FTMP2);
1297                                 s2 = REG_FTMP2;
1298                         }
1299                         M_FLTMOVE(s1, d);
1300                         M_FDIV(s2, d);
1301                         emit_store_dst(jd, iptr, d);
1302                         break;
1303
1304                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1305
1306                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1307                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1308                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1309                         if (s2 == d) {
1310                                 M_FLTMOVE(s2, REG_FTMP2);
1311                                 s2 = REG_FTMP2;
1312                         }
1313                         M_FLTMOVE(s1, d);
1314                         M_DDIV(s2, d);
1315                         emit_store_dst(jd, iptr, d);
1316                         break;
1317
1318                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1319
1320                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1321                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1322                         M_CVTIF(s1, d);
1323                         emit_store_dst(jd, iptr, d);
1324                         break;
1325
1326                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1327
1328                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1329                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1330                         M_CVTID(s1, d);
1331                         emit_store_dst(jd, iptr, d);
1332                         break;
1333
1334                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1335
1336                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1337                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1338                         M_CVTLF(s1, d);
1339                         emit_store_dst(jd, iptr, d);
1340                         break;
1341
1342                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1343
1344                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1345                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1346                         M_CVTLD(s1, d);
1347                         emit_store_dst(jd, iptr, d);
1348                         break;
1349
1350                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1351
1352                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1353                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1354                         M_CVTFI(s1, d);
1355                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1356                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1357                                 ((REG_RESULT == d) ? 0 : 3);
1358                         M_BNE(disp);
1359                         M_FLTMOVE(s1, REG_FTMP1);
1360                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1361                         M_CALL(REG_ITMP2);
1362                         M_INTMOVE(REG_RESULT, d);
1363                         emit_store_dst(jd, iptr, d);
1364                         break;
1365
1366                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1367
1368                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1369                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1370                         M_CVTDI(s1, d);
1371                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1372                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1373                                 ((REG_RESULT == d) ? 0 : 3);
1374                         M_BNE(disp);
1375                         M_FLTMOVE(s1, REG_FTMP1);
1376                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1377                         M_CALL(REG_ITMP2);
1378                         M_INTMOVE(REG_RESULT, d);
1379                         emit_store_dst(jd, iptr, d);
1380                         break;
1381
1382                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1383
1384                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1385                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1386                         M_CVTFL(s1, d);
1387                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1388                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1389                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1390                                 ((REG_RESULT == d) ? 0 : 3);
1391                         M_BNE(disp);
1392                         M_FLTMOVE(s1, REG_FTMP1);
1393                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1394                         M_CALL(REG_ITMP2);
1395                         M_INTMOVE(REG_RESULT, d);
1396                         emit_store_dst(jd, iptr, d);
1397                         break;
1398
1399                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1400
1401                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1402                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1403                         M_CVTDL(s1, d);
1404                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1405                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1406                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1407                                 ((REG_RESULT == d) ? 0 : 3);
1408                         M_BNE(disp);
1409                         M_FLTMOVE(s1, REG_FTMP1);
1410                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1411                         M_CALL(REG_ITMP2);
1412                         M_INTMOVE(REG_RESULT, d);
1413                         emit_store_dst(jd, iptr, d);
1414                         break;
1415
1416                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1417
1418                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1419                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1420                         M_CVTFD(s1, d);
1421                         emit_store_dst(jd, iptr, d);
1422                         break;
1423
1424                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1425
1426                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1427                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1428                         M_CVTDF(s1, d);
1429                         emit_store_dst(jd, iptr, d);
1430                         break;
1431
1432                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1433                                           /* == => 0, < => 1, > => -1 */
1434
1435                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1436                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1437                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1438                         M_CLR(d);
1439                         M_MOV_IMM(1, REG_ITMP1);
1440                         M_MOV_IMM(-1, REG_ITMP2);
1441                         emit_ucomiss_reg_reg(cd, s1, s2);
1442                         M_CMOVULT(REG_ITMP1, d);
1443                         M_CMOVUGT(REG_ITMP2, d);
1444                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1445                         emit_store_dst(jd, iptr, d);
1446                         break;
1447
1448                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1449                                           /* == => 0, < => 1, > => -1 */
1450
1451                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1452                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1453                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1454                         M_CLR(d);
1455                         M_MOV_IMM(1, REG_ITMP1);
1456                         M_MOV_IMM(-1, REG_ITMP2);
1457                         emit_ucomiss_reg_reg(cd, s1, s2);
1458                         M_CMOVULT(REG_ITMP1, d);
1459                         M_CMOVUGT(REG_ITMP2, d);
1460                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1461                         emit_store_dst(jd, iptr, d);
1462                         break;
1463
1464                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1465                                           /* == => 0, < => 1, > => -1 */
1466
1467                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1468                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1469                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1470                         M_CLR(d);
1471                         M_MOV_IMM(1, REG_ITMP1);
1472                         M_MOV_IMM(-1, REG_ITMP2);
1473                         emit_ucomisd_reg_reg(cd, s1, s2);
1474                         M_CMOVULT(REG_ITMP1, d);
1475                         M_CMOVUGT(REG_ITMP2, d);
1476                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1477                         emit_store_dst(jd, iptr, d);
1478                         break;
1479
1480                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1481                                           /* == => 0, < => 1, > => -1 */
1482
1483                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1484                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1485                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1486                         M_CLR(d);
1487                         M_MOV_IMM(1, REG_ITMP1);
1488                         M_MOV_IMM(-1, REG_ITMP2);
1489                         emit_ucomisd_reg_reg(cd, s1, s2);
1490                         M_CMOVULT(REG_ITMP1, d);
1491                         M_CMOVUGT(REG_ITMP2, d);
1492                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1493                         emit_store_dst(jd, iptr, d);
1494                         break;
1495
1496
1497                 /* memory operations **************************************************/
1498
1499                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1500
1501                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1502                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1503                         /* implicit null-pointer check */
1504                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1505                         emit_store_dst(jd, iptr, d);
1506                         break;
1507
1508                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1509
1510                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1511                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1512                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1513                         /* implicit null-pointer check */
1514                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1515                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1516                         emit_store_dst(jd, iptr, d);
1517                         break;
1518
1519                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1520
1521                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1522                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1523                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1524                         /* implicit null-pointer check */
1525                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1526                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1527                         emit_store_dst(jd, iptr, d);
1528                         break;
1529
1530                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1531
1532                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1533                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1534                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1535                         /* implicit null-pointer check */
1536                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1537                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1538                         emit_store_dst(jd, iptr, d);
1539                         break;
1540
1541                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1542
1543                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1544                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1545                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1546                         /* implicit null-pointer check */
1547                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1548                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1549                         emit_store_dst(jd, iptr, d);
1550                         break;
1551
1552                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1553
1554                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1555                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1556                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1557                         /* implicit null-pointer check */
1558                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1559                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1560                         emit_store_dst(jd, iptr, d);
1561                         break;
1562
1563                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1564
1565                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1566                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1567                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1568                         /* implicit null-pointer check */
1569                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1570                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1571                         emit_store_dst(jd, iptr, d);
1572                         break;
1573
1574                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1575
1576                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1577                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1578                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1579                         /* implicit null-pointer check */
1580                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1581                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1582                         emit_store_dst(jd, iptr, d);
1583                         break;
1584
1585                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1586
1587                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1588                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1589                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1590                         /* implicit null-pointer check */
1591                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1592                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1593                         emit_store_dst(jd, iptr, d);
1594                         break;
1595
1596
1597                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1598
1599                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1600                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1601                         /* implicit null-pointer check */
1602                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1603                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1604                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1605                         break;
1606
1607                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1608
1609                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1610                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1611                         /* implicit null-pointer check */
1612                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1613                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1614                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1615                         break;
1616
1617                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1618
1619                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1620                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1621                         /* implicit null-pointer check */
1622                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1623                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1624                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1625                         break;
1626
1627                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1628
1629                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1630                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1631                         /* implicit null-pointer check */
1632                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1633                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1634                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
1635                         break;
1636
1637                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1638
1639                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1640                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1641                         /* implicit null-pointer check */
1642                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1643                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1644                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
1645                         break;
1646
1647                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1648
1649                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1650                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1651                         /* implicit null-pointer check */
1652                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1653                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1654                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
1655                         break;
1656
1657                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1658
1659                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1660                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1661                         /* implicit null-pointer check */
1662                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1663                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1664                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
1665                         break;
1666
1667                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1668
1669                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1670                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1671                         /* implicit null-pointer check */
1672                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1673                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1674
1675                         M_MOV(s1, REG_A0);
1676                         M_MOV(s3, REG_A1);
1677                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
1678                         M_CALL(REG_ITMP1);
1679                         emit_exception_check(cd, iptr);
1680
1681                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1682                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1683                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1684                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1685                         break;
1686
1687
1688                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1689
1690                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1691                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1692                         /* implicit null-pointer check */
1693                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1694                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1695                         break;
1696
1697                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1698
1699                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1700                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1701                         /* implicit null-pointer check */
1702                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1703                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray, data[0]), s1, s2, 1);
1704                         break;
1705
1706                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1707
1708                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1709                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1710                         /* implicit null-pointer check */
1711                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1712                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1713                         break;
1714
1715                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1716
1717                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1718                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1719                         /* implicit null-pointer check */
1720                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1721                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray, data[0]), s1, s2, 2);
1722                         break;
1723
1724                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1725
1726                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1727                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1728                         /* implicit null-pointer check */
1729                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1730
1731                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1732                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1733                         }
1734                         else {
1735                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1736                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
1737                         }
1738                         break;
1739
1740                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1741
1742                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1743                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1744                         /* implicit null-pointer check */
1745                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1746                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1747                         break;
1748
1749
1750                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1751
1752                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1753                                 uf        = iptr->sx.s23.s3.uf;
1754                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1755                                 disp      = dseg_add_unique_address(cd, NULL);
1756                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1757
1758                                 /* must be calculated before codegen_add_patch_ref */
1759
1760                                 if (opt_shownops)
1761                                         disp -= PATCHER_CALL_SIZE;
1762
1763 /*                              PROFILE_CYCLE_STOP; */
1764
1765                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1766
1767 /*                              PROFILE_CYCLE_START; */
1768                         }
1769                         else {
1770                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1771                                 fieldtype = fi->type;
1772                                 disp      = dseg_add_address(cd, &(fi->value));
1773                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1774
1775                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1776                                         PROFILE_CYCLE_STOP;
1777
1778                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1779
1780                                         PROFILE_CYCLE_START;
1781
1782                                         if (opt_shownops)
1783                                                 disp -= PATCHER_CALL_SIZE;
1784                                 }
1785                         }
1786
1787                         /* This approach is much faster than moving the field
1788                            address inline into a register. */
1789
1790                         M_ALD(REG_ITMP1, RIP, disp);
1791
1792                         switch (fieldtype) {
1793                         case TYPE_INT:
1794                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1795                                 M_ILD(d, REG_ITMP1, 0);
1796                                 break;
1797                         case TYPE_LNG:
1798                         case TYPE_ADR:
1799                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1800                                 M_LLD(d, REG_ITMP1, 0);
1801                                 break;
1802                         case TYPE_FLT:
1803                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1804                                 M_FLD(d, REG_ITMP1, 0);
1805                                 break;
1806                         case TYPE_DBL:
1807                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1808                                 M_DLD(d, REG_ITMP1, 0);
1809                                 break;
1810                         }
1811                         emit_store_dst(jd, iptr, d);
1812                         break;
1813
1814                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1815
1816                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1817                                 uf        = iptr->sx.s23.s3.uf;
1818                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1819                                 disp      = dseg_add_unique_address(cd, NULL);
1820                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1821
1822                                 /* must be calculated before codegen_add_patch_ref */
1823
1824                                 if (opt_shownops)
1825                                         disp -= PATCHER_CALL_SIZE;
1826
1827 /*                              PROFILE_CYCLE_STOP; */
1828
1829                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1830
1831 /*                              PROFILE_CYCLE_START; */
1832                         }
1833                         else {
1834                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1835                                 fieldtype = fi->type;
1836                                 disp      = dseg_add_address(cd, &(fi->value));
1837                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1838
1839                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1840                                         PROFILE_CYCLE_STOP;
1841
1842                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1843
1844                                         PROFILE_CYCLE_START;
1845
1846                                         if (opt_shownops)
1847                                                 disp -= PATCHER_CALL_SIZE;
1848                                 }
1849                         }
1850
1851                         /* This approach is much faster than moving the field
1852                            address inline into a register. */
1853
1854                         M_ALD(REG_ITMP1, RIP, disp);
1855
1856                         switch (fieldtype) {
1857                         case TYPE_INT:
1858                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1859                                 M_IST(s1, REG_ITMP1, 0);
1860                                 break;
1861                         case TYPE_LNG:
1862                         case TYPE_ADR:
1863                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1864                                 M_LST(s1, REG_ITMP1, 0);
1865                                 break;
1866                         case TYPE_FLT:
1867                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1868                                 M_FST(s1, REG_ITMP1, 0);
1869                                 break;
1870                         case TYPE_DBL:
1871                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1872                                 M_DST(s1, REG_ITMP1, 0);
1873                                 break;
1874                         }
1875                         break;
1876
1877                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1878                                           /* val = value (in current instruction)     */
1879                                           /* following NOP)                           */
1880
1881                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1882                                 uf        = iptr->sx.s23.s3.uf;
1883                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1884                                 disp      = dseg_add_unique_address(cd, NULL);
1885                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1886
1887                                 /* must be calculated before codegen_add_patch_ref */
1888
1889                                 if (opt_shownops)
1890                                         disp -= PATCHER_CALL_SIZE;
1891
1892 /*                              PROFILE_CYCLE_STOP; */
1893
1894                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1895
1896 /*                              PROFILE_CYCLE_START; */
1897                         }
1898                         else {
1899                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1900                                 fieldtype = fi->type;
1901                                 disp      = dseg_add_address(cd, &(fi->value));
1902                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1903
1904                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1905                                         PROFILE_CYCLE_STOP;
1906
1907                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1908
1909                                         PROFILE_CYCLE_START;
1910
1911                                         if (opt_shownops)
1912                                                 disp -= PATCHER_CALL_SIZE;
1913                                 }
1914                         }
1915
1916                         /* This approach is much faster than moving the field
1917                            address inline into a register. */
1918
1919                         M_ALD(REG_ITMP1, RIP, disp);
1920
1921                         switch (fieldtype) {
1922                         case TYPE_INT:
1923                         case TYPE_FLT:
1924                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1925                                 break;
1926                         case TYPE_LNG:
1927                         case TYPE_ADR:
1928                         case TYPE_DBL:
1929                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1930                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1931                                 else {
1932                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1933                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1934                                 }
1935                                 break;
1936                         }
1937                         break;
1938
1939                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1940
1941                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1942
1943                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1944                                 uf        = iptr->sx.s23.s3.uf;
1945                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1946                                 disp      = 0;
1947
1948 /*                              PROFILE_CYCLE_STOP; */
1949
1950                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1951
1952 /*                              PROFILE_CYCLE_START; */
1953                         }
1954                         else {
1955                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1956                                 fieldtype = fi->type;
1957                                 disp      = fi->offset;
1958                         }
1959
1960                         /* implicit null-pointer check */
1961                         switch (fieldtype) {
1962                         case TYPE_INT:
1963                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1964                                 M_ILD32(d, s1, disp);
1965                                 break;
1966                         case TYPE_LNG:
1967                         case TYPE_ADR:
1968                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1969                                 M_LLD32(d, s1, disp);
1970                                 break;
1971                         case TYPE_FLT:
1972                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1973                                 M_FLD32(d, s1, disp);
1974                                 break;
1975                         case TYPE_DBL:
1976                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1977                                 M_DLD32(d, s1, disp);
1978                                 break;
1979                         }
1980                         emit_store_dst(jd, iptr, d);
1981                         break;
1982
1983                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1984
1985                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1986                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1987
1988                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1989                                 uf        = iptr->sx.s23.s3.uf;
1990                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1991                                 disp      = 0;
1992
1993 /*                              PROFILE_CYCLE_STOP; */
1994
1995                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1996
1997 /*                              PROFILE_CYCLE_START; */
1998                         }
1999                         else {
2000                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2001                                 fieldtype = fi->type;
2002                                 disp      = fi->offset;
2003                         }
2004
2005                         /* implicit null-pointer check */
2006                         switch (fieldtype) {
2007                         case TYPE_INT:
2008                                 M_IST32(s2, s1, disp);
2009                                 break;
2010                         case TYPE_LNG:
2011                         case TYPE_ADR:
2012                                 M_LST32(s2, s1, disp);
2013                                 break;
2014                         case TYPE_FLT:
2015                                 M_FST32(s2, s1, disp);
2016                                 break;
2017                         case TYPE_DBL:
2018                                 M_DST32(s2, s1, disp);
2019                                 break;
2020                         }
2021                         break;
2022
2023                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2024                                           /* val = value (in current instruction)     */
2025                                           /* following NOP)                           */
2026
2027                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2028
2029                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2030                                 uf        = iptr->sx.s23.s3.uf;
2031                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2032                                 disp      = 0;
2033
2034 /*                              PROFILE_CYCLE_STOP; */
2035
2036                                 codegen_add_patch_ref(cd, PATCHER_putfieldconst, uf, 0);
2037
2038 /*                              PROFILE_CYCLE_START; */
2039                         }
2040                         else {
2041                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2042                                 fieldtype = fi->type;
2043                                 disp      = fi->offset;
2044                         }
2045
2046                         /* implicit null-pointer check */
2047                         switch (fieldtype) {
2048                         case TYPE_INT:
2049                         case TYPE_FLT:
2050                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2051                                 break;
2052                         case TYPE_LNG:
2053                         case TYPE_ADR:
2054                         case TYPE_DBL:
2055                                 /* XXX why no check for IS_IMM32? */
2056                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2057                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2058                                 break;
2059                         }
2060                         break;
2061
2062
2063                 /* branch operations **************************************************/
2064
2065                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2066
2067                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2068                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2069
2070                         PROFILE_CYCLE_STOP;
2071
2072 #ifdef ENABLE_VERIFIER
2073                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2074                                 uc = iptr->sx.s23.s2.uc;
2075
2076                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2077                         }
2078 #endif /* ENABLE_VERIFIER */
2079
2080                         M_CALL_IMM(0);                            /* passing exception pc */
2081                         M_POP(REG_ITMP2_XPC);
2082
2083                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2084                         M_JMP(REG_ITMP3);
2085                         break;
2086
2087                 case ICMD_GOTO:         /* ... ==> ...                                */
2088                 case ICMD_RET:
2089
2090                         emit_br(cd, iptr->dst.block);
2091                         ALIGNCODENOP;
2092                         break;
2093
2094                 case ICMD_JSR:          /* ... ==> ...                                */
2095
2096                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2097                         ALIGNCODENOP;
2098                         break;
2099
2100                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2101                 case ICMD_IFNONNULL:
2102
2103                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2104                         M_TEST(s1);
2105                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2106                         break;
2107
2108                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2109                 case ICMD_IFLT:
2110                 case ICMD_IFLE:
2111                 case ICMD_IFNE:
2112                 case ICMD_IFGT:
2113                 case ICMD_IFGE:
2114
2115                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2116                         M_ICMP_IMM(iptr->sx.val.i, s1);
2117                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2118                         break;
2119
2120                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2121                 case ICMD_IF_LNE:
2122                 case ICMD_IF_LLT:
2123                 case ICMD_IF_LGE:
2124                 case ICMD_IF_LGT:
2125                 case ICMD_IF_LLE:
2126
2127                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2128                         if (IS_IMM32(iptr->sx.val.l))
2129                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2130                         else {
2131                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2132                                 M_LCMP(REG_ITMP2, s1);
2133                         }
2134                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2135                         break;
2136
2137                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2138                 case ICMD_IF_ICMPNE:
2139                 case ICMD_IF_ICMPLT:
2140                 case ICMD_IF_ICMPGE:
2141                 case ICMD_IF_ICMPGT:
2142                 case ICMD_IF_ICMPLE:
2143
2144                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2145                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2146                         M_ICMP(s2, s1);
2147                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2148                         break;
2149
2150                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2151                 case ICMD_IF_ACMPNE:
2152
2153                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2154                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2155                         M_LCMP(s2, s1);
2156                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2157                         break;
2158
2159                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2160                 case ICMD_IF_LCMPNE:
2161                 case ICMD_IF_LCMPLT:
2162                 case ICMD_IF_LCMPGE:
2163                 case ICMD_IF_LCMPGT:
2164                 case ICMD_IF_LCMPLE:
2165
2166                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2167                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2168                         M_LCMP(s2, s1);
2169                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2170                         break;
2171
2172                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2173                 case ICMD_LRETURN:
2174
2175                         REPLACEMENT_POINT_RETURN(cd, iptr);
2176                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2177                         M_INTMOVE(s1, REG_RESULT);
2178                         goto nowperformreturn;
2179
2180                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2181
2182                         REPLACEMENT_POINT_RETURN(cd, iptr);
2183                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2184                         M_INTMOVE(s1, REG_RESULT);
2185
2186 #ifdef ENABLE_VERIFIER
2187                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2188                                 uc = iptr->sx.s23.s2.uc;
2189
2190                                 PROFILE_CYCLE_STOP;
2191
2192                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2193
2194                                 PROFILE_CYCLE_START;
2195                         }
2196 #endif /* ENABLE_VERIFIER */
2197                         goto nowperformreturn;
2198
2199                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2200                 case ICMD_DRETURN:
2201
2202                         REPLACEMENT_POINT_RETURN(cd, iptr);
2203                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2204                         M_FLTMOVE(s1, REG_FRESULT);
2205                         goto nowperformreturn;
2206
2207                 case ICMD_RETURN:      /* ...  ==> ...                                */
2208
2209                         REPLACEMENT_POINT_RETURN(cd, iptr);
2210
2211 nowperformreturn:
2212                         {
2213                         s4 i, p;
2214
2215                         p = cd->stackframesize;
2216
2217 #if !defined(NDEBUG)
2218                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2219                                 emit_verbosecall_exit(jd);
2220 #endif /* !defined(NDEBUG) */
2221
2222 #if defined(ENABLE_THREADS)
2223                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2224                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2225
2226                                 /* we need to save the proper return value */
2227                                 switch (iptr->opc) {
2228                                 case ICMD_IRETURN:
2229                                 case ICMD_ARETURN:
2230                                 case ICMD_LRETURN:
2231                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2232                                         break;
2233                                 case ICMD_FRETURN:
2234                                 case ICMD_DRETURN:
2235                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2236                                         break;
2237                                 }
2238
2239                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2240                                 M_CALL(REG_ITMP1);
2241
2242                                 /* and now restore the proper return value */
2243                                 switch (iptr->opc) {
2244                                 case ICMD_IRETURN:
2245                                 case ICMD_ARETURN:
2246                                 case ICMD_LRETURN:
2247                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2248                                         break;
2249                                 case ICMD_FRETURN:
2250                                 case ICMD_DRETURN:
2251                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2252                                         break;
2253                                 }
2254                         }
2255 #endif
2256
2257                         /* restore saved registers */
2258
2259                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2260                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2261                         }
2262                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2263                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2264                         }
2265
2266                         /* deallocate stack */
2267
2268                         if (cd->stackframesize)
2269                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2270
2271                         /* generate method profiling code */
2272
2273                         PROFILE_CYCLE_STOP;
2274
2275                         M_RET;
2276                         }
2277                         break;
2278
2279
2280                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2281                         {
2282                                 s4 i, l;
2283                                 branch_target_t *table;
2284
2285                                 table = iptr->dst.table;
2286
2287                                 l = iptr->sx.s23.s2.tablelow;
2288                                 i = iptr->sx.s23.s3.tablehigh;
2289
2290                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2291                                 M_INTMOVE(s1, REG_ITMP1);
2292
2293                                 if (l != 0)
2294                                         M_ISUB_IMM(l, REG_ITMP1);
2295
2296                                 /* number of targets */
2297                                 i = i - l + 1;
2298
2299                 /* range check */
2300
2301                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2302                                 emit_bugt(cd, table[0].block);
2303
2304                                 /* build jump table top down and use address of lowest entry */
2305
2306                                 table += i;
2307
2308                                 while (--i >= 0) {
2309                                         dseg_add_target(cd, table->block);
2310                                         --table;
2311                                 }
2312
2313                                 /* length of dataseg after last dseg_add_target is used
2314                                    by load */
2315
2316                                 M_MOV_IMM(0, REG_ITMP2);
2317                                 dseg_adddata(cd);
2318                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2319                                 M_JMP(REG_ITMP1);
2320                         }
2321                         break;
2322
2323
2324                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2325                         {
2326                                 s4 i;
2327                                 lookup_target_t *lookup;
2328
2329                                 lookup = iptr->dst.lookup;
2330
2331                                 i = iptr->sx.s23.s2.lookupcount;
2332
2333                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2334                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2335
2336                                 while (--i >= 0) {
2337                                         M_ICMP_IMM(lookup->value, s1);
2338                                         emit_beq(cd, lookup->target.block);
2339                                         lookup++;
2340                                 }
2341
2342                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2343                                 ALIGNCODENOP;
2344                         }
2345                         break;
2346
2347
2348                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2349
2350                         bte = iptr->sx.s23.s3.bte;
2351                         md  = bte->md;
2352                         goto gen_method;
2353
2354                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2355
2356                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2357                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2358                 case ICMD_INVOKEINTERFACE:
2359
2360                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2361
2362                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2363                                 lm = NULL;
2364                                 um = iptr->sx.s23.s3.um;
2365                                 md = um->methodref->parseddesc.md;
2366                         }
2367                         else {
2368                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2369                                 um = NULL;
2370                                 md = lm->parseddesc;
2371                         }
2372
2373 gen_method:
2374                         s3 = md->paramcount;
2375
2376                         MCODECHECK((20 * s3) + 128);
2377
2378                         /* copy arguments to registers or stack location */
2379
2380                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2381                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2382                                 d   = md->params[s3].regoff;
2383
2384                                 /* already preallocated (ARGVAR)? */
2385
2386                                 if (var->flags & PREALLOC)
2387                                         continue;
2388
2389                                 if (IS_INT_LNG_TYPE(var->type)) {
2390                                         if (!md->params[s3].inmemory) {
2391                                                 s1 = emit_load(jd, iptr, var, d);
2392                                                 M_INTMOVE(s1, d);
2393                                         }
2394                                         else {
2395                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2396                                                 M_LST(s1, REG_SP, d * 8);
2397                                         }
2398                                 }
2399                                 else {
2400                                         if (!md->params[s3].inmemory) {
2401                                                 s1 = emit_load(jd, iptr, var, d);
2402                                                 M_FLTMOVE(s1, d);
2403                                         }
2404                                         else {
2405                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2406
2407                                                 if (IS_2_WORD_TYPE(var->type))
2408                                                         M_DST(s1, REG_SP, d * 8);
2409                                                 else
2410                                                         M_FST(s1, REG_SP, d * 8);
2411                                         }
2412                                 }
2413                         }
2414
2415                         /* generate method profiling code */
2416
2417                         PROFILE_CYCLE_STOP;
2418
2419                         switch (iptr->opc) {
2420                         case ICMD_BUILTIN:
2421                                 M_MOV_IMM(bte->fp, REG_ITMP1);
2422                                 M_CALL(REG_ITMP1);
2423
2424                                 emit_exception_check(cd, iptr);
2425                                 break;
2426
2427                         case ICMD_INVOKESPECIAL:
2428                                 emit_nullpointer_check(cd, iptr, REG_A0);
2429                                 /* fall through */
2430
2431                         case ICMD_INVOKESTATIC:
2432                                 if (lm == NULL) {
2433                                         disp = dseg_add_unique_address(cd, NULL);
2434                                         disp = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
2435
2436                                         /* must be calculated before codegen_add_patch_ref */
2437
2438                                         if (opt_shownops)
2439                                                 disp -= PATCHER_CALL_SIZE;
2440
2441                                         codegen_add_patch_ref(cd, PATCHER_invokestatic_special,
2442                                                                                   um, disp);
2443
2444 /*                                      a = 0; */
2445                                 }
2446                                 else {
2447                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2448                                         disp = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
2449
2450 /*                                      a = (ptrint) lm->stubroutine; */
2451                                 }
2452
2453 /*                              M_MOV_IMM(a, REG_ITMP2); */
2454                                 M_ALD(REG_ITMP2, RIP, disp);
2455                                 M_CALL(REG_ITMP2);
2456                                 break;
2457
2458                         case ICMD_INVOKEVIRTUAL:
2459                                 if (lm == NULL) {
2460                                         codegen_add_patch_ref(cd, PATCHER_invokevirtual, um, 0);
2461
2462                                         s1 = 0;
2463                                 }
2464                                 else {
2465                                         s1 = OFFSET(vftbl_t, table[0]) +
2466                                                 sizeof(methodptr) * lm->vftblindex;
2467                                 }
2468
2469                                 /* implicit null-pointer check */
2470                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_objectheader, vftbl));
2471                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2472                                 M_CALL(REG_ITMP3);
2473                                 break;
2474
2475                         case ICMD_INVOKEINTERFACE:
2476                                 if (lm == NULL) {
2477                                         codegen_add_patch_ref(cd, PATCHER_invokeinterface, um, 0);
2478
2479                                         s1 = 0;
2480                                         s2 = 0;
2481                                 }
2482                                 else {
2483                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2484                                                 sizeof(methodptr) * lm->class->index;
2485
2486                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2487                                 }
2488
2489                                 /* implicit null-pointer check */
2490                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_objectheader, vftbl));
2491                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2492                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2493                                 M_CALL(REG_ITMP3);
2494                                 break;
2495                         }
2496
2497                         /* generate method profiling code */
2498
2499                         PROFILE_CYCLE_START;
2500
2501                         /* store size of call code in replacement point */
2502
2503                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2504
2505                         /* store return value */
2506
2507                         switch (md->returntype.type) {
2508                         case TYPE_INT:
2509                         case TYPE_LNG:
2510                         case TYPE_ADR:
2511                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2512                                 M_INTMOVE(REG_RESULT, s1);
2513                                 emit_store_dst(jd, iptr, s1);
2514                                 break;
2515                         case TYPE_FLT:
2516                         case TYPE_DBL:
2517                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2518                                 M_FLTMOVE(REG_FRESULT, s1);
2519                                 emit_store_dst(jd, iptr, s1);
2520                                 break;
2521                         default:
2522                                 /* TYPE_VOID */
2523                                 break;
2524                         }
2525                         break;
2526
2527
2528                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2529
2530                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2531                                 /* object type cast-check */
2532
2533                                 classinfo *super;
2534                                 vftbl_t   *supervftbl;
2535                                 s4         superindex;
2536
2537                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2538                                         super      = NULL;
2539                                         superindex = 0;
2540                                         supervftbl = NULL;
2541                                 }
2542                                 else {
2543                                         super      = iptr->sx.s23.s3.c.cls;
2544                                         superindex = super->index;
2545                                         supervftbl = super->vftbl;
2546                                 }
2547
2548 #if defined(ENABLE_THREADS)
2549                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
2550 #endif
2551                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2552
2553                                 /* if class is not resolved, check which code to call */
2554
2555                                 if (super == NULL) {
2556                                         M_TEST(s1);
2557                                         emit_label_beq(cd, BRANCH_LABEL_1);
2558
2559                                         codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2560                                                                                   iptr->sx.s23.s3.c.ref, 0);
2561
2562                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2563                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2564                                         emit_label_beq(cd, BRANCH_LABEL_2);
2565                                 }
2566
2567                                 /* interface checkcast code */
2568
2569                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2570                                         if (super != NULL) {
2571                                                 M_TEST(s1);
2572                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2573                                         }
2574
2575                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2576
2577                                         if (super == NULL) {
2578                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_interface,
2579                                                                                           iptr->sx.s23.s3.c.ref,
2580                                                                                           0);
2581                                         }
2582
2583                                         M_ILD32(REG_ITMP3,
2584                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2585                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2586                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2587
2588                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2589                                                         OFFSET(vftbl_t, interfacetable[0]) -
2590                                                         superindex * sizeof(methodptr*));
2591                                         M_TEST(REG_ITMP3);
2592                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2593
2594                                         if (super == NULL)
2595                                                 emit_label_br(cd, BRANCH_LABEL_4);
2596                                         else
2597                                                 emit_label(cd, BRANCH_LABEL_3);
2598                                 }
2599
2600                                 /* class checkcast code */
2601
2602                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2603                                         if (super == NULL) {
2604                                                 emit_label(cd, BRANCH_LABEL_2);
2605                                         }
2606                                         else {
2607                                                 M_TEST(s1);
2608                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2609                                         }
2610
2611                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2612
2613                                         if (super == NULL) {
2614                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_class,
2615                                                                                           iptr->sx.s23.s3.c.ref,
2616                                                                                           0);
2617                                         }
2618
2619                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2620 #if defined(ENABLE_THREADS)
2621                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
2622 #endif
2623                                         M_ILD32(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2624
2625                                         /*                                      if (s1 != REG_ITMP1) { */
2626                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2627                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2628                                         /*                                                                                              REG_ITMP1); */
2629                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2630                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2631                                         /*                                                                                              REG_ITMP3); */
2632                                         /*  #if defined(ENABLE_THREADS) */
2633                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2634                                         /*  #endif */
2635                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2636
2637                                         /*                                      } else { */
2638
2639                                         M_ILD32(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
2640                                         M_ISUB(REG_ITMP3, REG_ITMP2);
2641                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2642                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2643                                         /*                                      } */
2644 #if defined(ENABLE_THREADS)
2645                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
2646 #endif
2647                                         M_ICMP(REG_ITMP3, REG_ITMP2);
2648                                         emit_classcast_check(cd, iptr, BRANCH_UGT, REG_ITMP3, s1);
2649
2650                                         if (super != NULL)
2651                                                 emit_label(cd, BRANCH_LABEL_5);
2652                                 }
2653
2654                                 if (super == NULL) {
2655                                         emit_label(cd, BRANCH_LABEL_1);
2656                                         emit_label(cd, BRANCH_LABEL_4);
2657                                 }
2658
2659                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2660                         }
2661                         else {
2662                                 /* array type cast-check */
2663
2664                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2665                                 M_INTMOVE(s1, REG_A0);
2666
2667                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2668                                         codegen_add_patch_ref(cd, PATCHER_builtin_arraycheckcast,
2669                                                                                   iptr->sx.s23.s3.c.ref, 0);
2670                                 }
2671
2672                                 M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2673                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2674                                 M_CALL(REG_ITMP1);
2675
2676                                 /* s1 may have been destroyed over the function call */
2677                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2678                                 M_TEST(REG_RESULT);
2679                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2680
2681                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2682                         }
2683
2684                         M_INTMOVE(s1, d);
2685                         emit_store_dst(jd, iptr, d);
2686                         break;
2687
2688                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2689
2690                         {
2691                         classinfo *super;
2692                         vftbl_t   *supervftbl;
2693                         s4         superindex;
2694
2695                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2696                                 super      = NULL;
2697                                 superindex = 0;
2698                                 supervftbl = NULL;
2699
2700                         } else {
2701                                 super      = iptr->sx.s23.s3.c.cls;
2702                                 superindex = super->index;
2703                                 supervftbl = super->vftbl;
2704                         }
2705
2706 #if defined(ENABLE_THREADS)
2707             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
2708 #endif
2709
2710                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2711                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2712
2713                         if (s1 == d) {
2714                                 M_INTMOVE(s1, REG_ITMP1);
2715                                 s1 = REG_ITMP1;
2716                         }
2717
2718                         M_CLR(d);
2719
2720                         /* if class is not resolved, check which code to call */
2721
2722                         if (super == NULL) {
2723                                 M_TEST(s1);
2724                                 emit_label_beq(cd, BRANCH_LABEL_1);
2725
2726                                 codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2727                                                                           iptr->sx.s23.s3.c.ref, 0);
2728
2729                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2730                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2731                                 emit_label_beq(cd, BRANCH_LABEL_2);
2732                         }
2733
2734                         /* interface instanceof code */
2735
2736                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2737                                 if (super != NULL) {
2738                                         M_TEST(s1);
2739                                         emit_label_beq(cd, BRANCH_LABEL_3);
2740                                 }
2741
2742                                 M_ALD(REG_ITMP1, s1, OFFSET(java_objectheader, vftbl));
2743
2744                                 if (super == NULL) {
2745                                         codegen_add_patch_ref(cd, PATCHER_instanceof_interface,
2746                                                                                   iptr->sx.s23.s3.c.ref, 0);
2747                                 }
2748
2749                                 M_ILD32(REG_ITMP3,
2750                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2751                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2752
2753                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2754
2755                                 M_BLE(a);
2756                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2757                                                 OFFSET(vftbl_t, interfacetable[0]) -
2758                                                 superindex * sizeof(methodptr*));
2759                                 M_TEST(REG_ITMP1);
2760                                 M_SETNE(d);
2761
2762                                 if (super == NULL)
2763                                         emit_label_br(cd, BRANCH_LABEL_4);
2764                                 else
2765                                         emit_label(cd, BRANCH_LABEL_3);
2766                         }
2767
2768                         /* class instanceof code */
2769
2770                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2771                                 if (super == NULL) {
2772                                         emit_label(cd, BRANCH_LABEL_2);
2773                                 }
2774                                 else {
2775                                         M_TEST(s1);
2776                                         emit_label_beq(cd, BRANCH_LABEL_5);
2777                                 }
2778
2779                                 M_ALD(REG_ITMP1, s1, OFFSET(java_objectheader, vftbl));
2780
2781                                 if (super == NULL) {
2782                                         codegen_add_patch_ref(cd, PATCHER_instanceof_class,
2783                                                                                   iptr->sx.s23.s3.c.ref, 0);
2784                                 }
2785
2786                                 M_MOV_IMM(supervftbl, REG_ITMP2);
2787
2788 #if defined(ENABLE_THREADS)
2789                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
2790 #endif
2791
2792                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
2793                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2794                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2795
2796 #if defined(ENABLE_THREADS)
2797                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
2798 #endif
2799
2800                                 M_ISUB(REG_ITMP2, REG_ITMP1);
2801                                 M_CLR(d); /* may be REG_ITMP2 */
2802                                 M_ICMP(REG_ITMP3, REG_ITMP1);
2803                                 M_SETULE(d);
2804
2805                                 if (super != NULL)
2806                                         emit_label(cd, BRANCH_LABEL_5);
2807                         }
2808
2809                         if (super == NULL) {
2810                                 emit_label(cd, BRANCH_LABEL_1);
2811                                 emit_label(cd, BRANCH_LABEL_4);
2812                         }
2813
2814                         emit_store_dst(jd, iptr, d);
2815                         }
2816                         break;
2817
2818                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2819
2820                         /* check for negative sizes and copy sizes to stack if necessary  */
2821
2822                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2823
2824                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2825
2826                                 /* copy SAVEDVAR sizes to stack */
2827                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2828
2829                                 /* Already Preallocated? */
2830                                 if (!(var->flags & PREALLOC)) {
2831                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2832                                         M_LST(s2, REG_SP, s1 * 8);
2833                                 }
2834                         }
2835
2836                         /* is a patcher function set? */
2837
2838                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2839                                 codegen_add_patch_ref(cd, PATCHER_builtin_multianewarray,
2840                                                                           iptr->sx.s23.s3.c.ref, 0);
2841                         }
2842
2843                         /* a0 = dimension count */
2844
2845                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2846
2847                         /* a1 = classinfo */
2848
2849                         M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2850
2851                         /* a2 = pointer to dimensions = stack pointer */
2852
2853                         M_MOV(REG_SP, REG_A2);
2854
2855                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2856                         M_CALL(REG_ITMP1);
2857
2858                         /* check for exception before result assignment */
2859
2860                         emit_exception_check(cd, iptr);
2861
2862                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2863                         M_INTMOVE(REG_RESULT, s1);
2864                         emit_store_dst(jd, iptr, s1);
2865                         break;
2866
2867                 default:
2868                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2869                                                                                    iptr->opc);
2870                         return false;
2871         } /* switch */
2872
2873         } /* for instruction */
2874
2875         MCODECHECK(512); /* XXX require a lower number? */
2876
2877         /* At the end of a basic block we may have to append some nops,
2878            because the patcher stub calling code might be longer than the
2879            actual instruction. So codepatching does not change the
2880            following block unintentionally. */
2881
2882         if (cd->mcodeptr < cd->lastmcodeptr) {
2883                 while (cd->mcodeptr < cd->lastmcodeptr) {
2884                         M_NOP;
2885                 }
2886         }
2887
2888         } /* if (bptr -> flags >= BBREACHED) */
2889         } /* for basic block */
2890
2891         dseg_createlinenumbertable(cd);
2892
2893         /* generate stubs */
2894
2895         emit_patcher_stubs(jd);
2896         REPLACEMENT_EMIT_STUBS(jd);
2897
2898         /* everything's ok */
2899
2900         return true;
2901 }
2902
2903
2904 /* codegen_emit_stub_compiler **************************************************
2905
2906    Emit a stub routine which calls the compiler.
2907
2908 *******************************************************************************/
2909
2910 void codegen_emit_stub_compiler(jitdata *jd)
2911 {
2912         methodinfo  *m;
2913         codegendata *cd;
2914
2915         /* get required compiler data */
2916
2917         m  = jd->m;
2918         cd = jd->cd;
2919
2920         /* code for the stub */
2921
2922         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
2923         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
2924         M_JMP(REG_ITMP3);
2925 }
2926
2927
2928 /* codegen_emit_stub_native ****************************************************
2929
2930    Emits a stub routine which calls a native method.
2931
2932 *******************************************************************************/
2933
2934 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f)
2935 {
2936         methodinfo  *m;
2937         codeinfo    *code;
2938         codegendata *cd;
2939         methoddesc  *md;
2940         s4           nativeparams;
2941         s4           i, j;
2942         s4           t;
2943         s4           s1, s2;
2944
2945         /* get required compiler data */
2946
2947         m    = jd->m;
2948         code = jd->code;
2949         cd   = jd->cd;
2950
2951         /* initialize variables */
2952
2953         md = m->parseddesc;
2954         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
2955
2956         /* calculate stack frame size */
2957
2958         cd->stackframesize =
2959                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
2960                 sizeof(localref_table) / SIZEOF_VOID_P +
2961                 md->paramcount +
2962                 1 +                       /* functionptr, TODO: store in data segment */
2963                 nmd->memuse;
2964
2965         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
2966
2967         /* create method header */
2968
2969         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2970         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2971         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
2972         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2973         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2974         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2975         (void) dseg_addlinenumbertablesize(cd);
2976         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
2977
2978 #if defined(ENABLE_PROFILING)
2979         /* generate native method profiling code */
2980
2981         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
2982                 /* count frequency */
2983
2984                 M_MOV_IMM(code, REG_ITMP3);
2985                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
2986         }
2987 #endif
2988
2989         /* generate stub code */
2990
2991         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2992
2993 #if !defined(NDEBUG)
2994         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2995                 emit_verbosecall_enter(jd);
2996 #endif
2997
2998         /* get function address (this must happen before the stackframeinfo) */
2999
3000 #if !defined(WITH_STATIC_CLASSPATH)
3001         if (f == NULL)
3002                 codegen_add_patch_ref(cd, PATCHER_resolve_native, m, 0);
3003 #endif
3004
3005         M_MOV_IMM(f, REG_ITMP3);
3006
3007         /* save integer and float argument registers */
3008
3009         for (i = 0; i < md->paramcount; i++) {
3010                 if (!md->params[i].inmemory) {
3011                         s1 = md->params[i].regoff;
3012
3013                         switch (md->paramtypes[i].type) {
3014                         case TYPE_INT:
3015                         case TYPE_LNG:
3016                         case TYPE_ADR:
3017                                 M_LST(s1, REG_SP, i * 8);
3018                                 break;
3019                         case TYPE_FLT:
3020                         case TYPE_DBL:
3021                                 M_DST(s1, REG_SP, i * 8);
3022                                 break;
3023                         }
3024                 }
3025         }
3026
3027         M_AST(REG_ITMP3, REG_SP, md->paramcount * 8);
3028
3029         /* create dynamic stack info */
3030
3031         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3032         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3033         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3034         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3035         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3036         M_CALL(REG_ITMP1);
3037
3038         /* restore integer and float argument registers */
3039
3040         for (i = 0; i < md->paramcount; i++) {
3041                 if (!md->params[i].inmemory) {
3042                         s1 = md->params[i].regoff;
3043
3044                         switch (md->paramtypes[i].type) {
3045                         case TYPE_INT:
3046                         case TYPE_LNG:
3047                         case TYPE_ADR:
3048                                 M_LLD(s1, REG_SP, i * 8);
3049                                 break;
3050                         case TYPE_FLT:
3051                         case TYPE_DBL:
3052                                 M_DLD(s1, REG_SP, i * 8);
3053                                 break;
3054                         }
3055                 }
3056         }
3057
3058         M_ALD(REG_ITMP3, REG_SP, md->paramcount * 8);
3059
3060         /* copy or spill arguments to new locations */
3061
3062         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3063                 t  = md->paramtypes[i].type;
3064                 s2 = nmd->params[j].regoff;
3065
3066                 if (IS_INT_LNG_TYPE(t)) {
3067                         if (!md->params[i].inmemory) {
3068                                 s1 = md->params[i].regoff;
3069
3070                                 if (!nmd->params[j].inmemory)
3071                                         M_INTMOVE(s1, s2);
3072                                 else
3073                                         M_LST(s1, REG_SP, s2 * 8);
3074                         }
3075                         else {
3076                                 s1 = md->params[i].regoff + cd->stackframesize + 1;/* +1 (RA) */
3077                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
3078                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
3079                         }
3080                 }
3081                 else {
3082                         /* We only copy spilled float arguments, as the float
3083                            argument registers keep unchanged. */
3084
3085                         if (md->params[i].inmemory) {
3086                                 s1 = md->params[i].regoff + cd->stackframesize + 1;/* +1 (RA) */
3087
3088                                 if (IS_2_WORD_TYPE(t)) {
3089                                         M_DLD(REG_FTMP1, REG_SP, s1 * 8);
3090                                         M_DST(REG_FTMP1, REG_SP, s2 * 8);
3091                                 }
3092                                 else {
3093                                         M_FLD(REG_FTMP1, REG_SP, s1 * 8);
3094                                         M_FST(REG_FTMP1, REG_SP, s2 * 8);
3095                                 }
3096                         }
3097                 }
3098         }
3099
3100         /* put class into second argument register */
3101
3102         if (m->flags & ACC_STATIC)
3103                 M_MOV_IMM(m->class, REG_A1);
3104
3105         /* put env into first argument register */
3106
3107         M_MOV_IMM(_Jv_env, REG_A0);
3108
3109         /* do the native function call */
3110
3111         M_CALL(REG_ITMP3);
3112
3113         /* save return value */
3114
3115         switch (md->returntype.type) {
3116         case TYPE_INT:
3117         case TYPE_LNG:
3118         case TYPE_ADR:
3119                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3120                 break;
3121         case TYPE_FLT:
3122         case TYPE_DBL:
3123                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3124                 break;
3125         case TYPE_VOID:
3126                 break;
3127         }
3128
3129 #if !defined(NDEBUG)
3130         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3131                 emit_verbosecall_exit(jd);
3132 #endif
3133
3134         /* remove native stackframe info */
3135
3136         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3137         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3138         M_CALL(REG_ITMP1);
3139         M_MOV(REG_RESULT, REG_ITMP3);
3140
3141         /* restore return value */
3142
3143         switch (md->returntype.type) {
3144         case TYPE_INT:
3145         case TYPE_LNG:
3146         case TYPE_ADR:
3147                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3148                 break;
3149         case TYPE_FLT:
3150         case TYPE_DBL:
3151                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3152                 break;
3153         case TYPE_VOID:
3154                 break;
3155         }
3156
3157         /* remove stackframe */
3158
3159         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3160
3161         /* test for exception */
3162
3163         M_TEST(REG_ITMP3);
3164         M_BNE(1);
3165         M_RET;
3166
3167         /* handle exception */
3168
3169         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3170         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3171         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3172
3173         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3174         M_JMP(REG_ITMP3);
3175
3176         /* generate patcher stubs */
3177
3178         emit_patcher_stubs(jd);
3179 }
3180
3181
3182 /*
3183  * These are local overrides for various environment variables in Emacs.
3184  * Please do not remove this and leave it at the end of the file, where
3185  * Emacs will automagically detect them.
3186  * ---------------------------------------------------------------------
3187  * Local variables:
3188  * mode: c
3189  * indent-tabs-mode: t
3190  * c-basic-offset: 4
3191  * tab-width: 4
3192  * End:
3193  * vim:noexpandtab:sw=4:ts=4:
3194  */