src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25 */
  26
  27
  28 #include "config.h"
  29
  30 #include <assert.h>
  31 #include <stdio.h>
  32 #include <stdint.h>
  33
  34 #include "vm/types.h"
  35
  36 #include "md-abi.h"
  37
  38 #include "vm/jit/x86_64/arch.h"
  39 #include "vm/jit/x86_64/codegen.h"
  40 #include "vm/jit/x86_64/emit.h"
  41
  42 #include "mm/memory.h"
  43
  44 #include "native/jni.h"
  45 #include "native/localref.h"
  46 #include "native/native.h"
  47
  48 #include "threads/lock-common.h"
  49
  50 #include "vm/builtin.h"
  51 #include "vm/exceptions.h"
  52 #include "vm/global.h"
  53 #include "vm/stringlocal.h"
  54 #include "vm/vm.h"
  55
  56 #include "vm/jit/abi.h"
  57 #include "vm/jit/asmpart.h"
  58 #include "vm/jit/code.h"
  59 #include "vm/jit/codegen-common.h"
  60 #include "vm/jit/dseg.h"
  61 #include "vm/jit/emit-common.h"
  62 #include "vm/jit/jit.h"
  63 #include "vm/jit/linenumbertable.h"
  64 #include "vm/jit/methodheader.h"
  65 #include "vm/jit/parse.h"
  66 #include "vm/jit/patcher-common.h"
  67 #include "vm/jit/reg.h"
  68 #include "vm/jit/replace.h"
  69 #include "vm/jit/stacktrace.h"
  70
  71 #if defined(ENABLE_LSRA)
  72 # include "vm/jit/allocator/lsra.h"
  73 #endif
  74
  75 #include "vmcore/loader.h"
  76 #include "vmcore/options.h"
  77 #include "vmcore/statistics.h"
  78
  79
  80 /* codegen_emit ****************************************************************
  81
  82    Generates machine code.
  83
  84 *******************************************************************************/
  85
  86 bool codegen_emit(jitdata *jd)
  87 {
  88         methodinfo         *m;
  89         codeinfo           *code;
  90         codegendata        *cd;
  91         registerdata       *rd;
  92         s4                  len, s1, s2, s3, d, disp;
  93         u2                  currentline;
  94         ptrint              a;
  95         varinfo            *var, *dst;
  96         basicblock         *bptr;
  97         instruction        *iptr;
  98         constant_classref  *cr;
  99         unresolved_class   *uc;
 100         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 101         unresolved_method  *um;
 102         builtintable_entry *bte;
 103         methoddesc         *md;
 104         fieldinfo          *fi;
 105         unresolved_field   *uf;
 106         s4                  fieldtype;
 107         s4                 varindex;
 108
 109         /* get required compiler data */
 110
 111         m    = jd->m;
 112         code = jd->code;
 113         cd   = jd->cd;
 114         rd   = jd->rd;
 115
 116         /* prevent compiler warnings */
 117
 118         d   = 0;
 119         lm  = NULL;
 120         um  = NULL;
 121         bte = NULL;
 122
 123         {
 124         s4 i, p, t, l;
 125         s4 savedregs_num;
 126
 127         savedregs_num = 0;
 128
 129         /* space to save used callee saved registers */
 130
 131         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 132         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 133
 134         cd->stackframesize = rd->memuse + savedregs_num;
 135
 136 #if defined(ENABLE_THREADS)
 137         /* space to save argument of monitor_enter */
 138
 139         if (checksync && code_is_synchronized(code))
 140                 cd->stackframesize++;
 141 #endif
 142
 143         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 144            native code e.g. libc or jni (alignment problems with
 145            movaps). */
 146
 147         if (!code_is_leafmethod(code) || opt_verbosecall)
 148                 cd->stackframesize |= 0x1;
 149
 150         /* create method header */
 151
 152         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 153         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 154
 155         code->synchronizedoffset = rd->memuse * 8;
 156
 157         if (code_is_leafmethod(code))
 158                 (void) dseg_add_unique_s4(cd, 1);                  /* IsLeaf          */
 159         else
 160                 (void) dseg_add_unique_s4(cd, 0);                  /* IsLeaf          */
 161
 162         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 163         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 164
 165 #if defined(ENABLE_PROFILING)
 166         /* generate method profiling code */
 167
 168         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 169                 /* count frequency */
 170
 171                 M_MOV_IMM(code, REG_ITMP3);
 172                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 173
 174                 PROFILE_CYCLE_START;
 175         }
 176 #endif
 177
 178         /* create stack frame (if necessary) */
 179
 180         if (cd->stackframesize)
 181                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 182
 183         /* save used callee saved registers */
 184
 185         p = cd->stackframesize;
 186         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 187                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 188         }
 189         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 190                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 191         }
 192
 193         /* take arguments out of register or stack frame */
 194
 195         md = m->parseddesc;
 196
 197         for (p = 0, l = 0; p < md->paramcount; p++) {
 198                 t = md->paramtypes[p].type;
 199
 200                 varindex = jd->local_map[l * 5 + t];
 201
 202                 l++;
 203                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 204                         l++;
 205
 206                 if (varindex == UNUSED)
 207                         continue;
 208
 209                 var = VAR(varindex);
 210
 211                 s1 = md->params[p].regoff;
 212
 213                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 214                         if (!md->params[p].inmemory) {           /* register arguments    */
 215                                 if (!IS_INMEMORY(var->flags))
 216                                         M_INTMOVE(s1, var->vv.regoff);
 217                                 else
 218                                     M_LST(s1, REG_SP, var->vv.regoff);
 219                         }
 220                         else {                                 /* stack arguments       */
 221                                 if (!IS_INMEMORY(var->flags))
 222                                         /* + 8 for return address */
 223                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 224                                 else
 225                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 226                         }
 227                 }
 228                 else {                                     /* floating args         */
 229                         if (!md->params[p].inmemory) {           /* register arguments    */
 230                                 if (!IS_INMEMORY(var->flags))
 231                                         M_FLTMOVE(s1, var->vv.regoff);
 232                                 else
 233                                         M_DST(s1, REG_SP, var->vv.regoff);
 234                         }
 235                         else {                                 /* stack arguments       */
 236                                 if (!IS_INMEMORY(var->flags))
 237                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 238                                 else
 239                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 240                         }
 241                 }
 242         }
 243
 244         /* save monitorenter argument */
 245
 246 #if defined(ENABLE_THREADS)
 247         if (checksync && code_is_synchronized(code)) {
 248                 /* stack offset for monitor argument */
 249
 250                 s1 = rd->memuse;
 251
 252                 if (opt_verbosecall) {
 253                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 254
 255                         for (p = 0; p < INT_ARG_CNT; p++)
 256                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 257
 258                         for (p = 0; p < FLT_ARG_CNT; p++)
 259                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 260
 261                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 262                 }
 263
 264                 /* decide which monitor enter function to call */
 265
 266                 if (m->flags & ACC_STATIC) {
 267                         M_MOV_IMM(&m->clazz->object.header, REG_A0);
 268                 }
 269                 else {
 270                         M_TEST(REG_A0);
 271                         M_BNE(8);
 272                         M_ALD_MEM(REG_A0, EXCEPTION_HARDWARE_NULLPOINTER);
 273                 }
 274
 275                 M_AST(REG_A0, REG_SP, s1 * 8);
 276                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 277                 M_CALL(REG_ITMP1);
 278
 279                 if (opt_verbosecall) {
 280                         for (p = 0; p < INT_ARG_CNT; p++)
 281                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 282
 283                         for (p = 0; p < FLT_ARG_CNT; p++)
 284                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 285
 286                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 287                 }
 288         }
 289 #endif
 290
 291 #if !defined(NDEBUG)
 292         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 293                 emit_verbosecall_enter(jd);
 294 #endif /* !defined(NDEBUG) */
 295
 296         }
 297
 298         /* end of header generation */
 299
 300         /* create replacement points */
 301
 302         REPLACEMENT_POINTS_INIT(cd, jd);
 303
 304         /* walk through all basic blocks */
 305
 306         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 307
 308                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 309
 310                 if (bptr->flags >= BBREACHED) {
 311
 312                 /* branch resolving */
 313
 314                 codegen_resolve_branchrefs(cd, bptr);
 315
 316                 /* handle replacement points */
 317
 318                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 319
 320                 /* copy interface registers to their destination */
 321
 322                 len = bptr->indepth;
 323                 MCODECHECK(512);
 324
 325 #if defined(ENABLE_PROFILING)
 326                 /* generate basicblock profiling code */
 327
 328                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 329                         /* count frequency */
 330
 331                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 332                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 333
 334                         /* if this is an exception handler, start profiling again */
 335
 336                         if (bptr->type == BBTYPE_EXH)
 337                                 PROFILE_CYCLE_START;
 338                 }
 339 #endif
 340
 341 #if defined(ENABLE_LSRA)
 342                 if (opt_lsra) {
 343                         while (len) {
 344                                 len--;
 345                                 src = bptr->invars[len];
 346                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 347                                         if (bptr->type == BBTYPE_EXH) {
 348 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 349                                                 if (!IS_INMEMORY(src->flags))
 350                                                         d= src->vv.regoff;
 351                                                 else
 352                                                         d=REG_ITMP1;
 353                                                 M_INTMOVE(REG_ITMP1, d);
 354                                                 emit_store(jd, NULL, src, d);
 355                                         }
 356                                 }
 357                         }
 358
 359                 } else {
 360 #endif
 361
 362                 while (len) {
 363                         len--;
 364                         var = VAR(bptr->invars[len]);
 365                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 366                                 if (bptr->type == BBTYPE_EXH) {
 367                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 368                                         M_INTMOVE(REG_ITMP1, d);
 369                                         emit_store(jd, NULL, var, d);
 370                                 }
 371                         }
 372                         else {
 373                                 assert((var->flags & INOUT));
 374                         }
 375                 }
 376 #if defined(ENABLE_LSRA)
 377                 }
 378 #endif
 379                 /* walk through all instructions */
 380
 381                 len = bptr->icount;
 382                 currentline = 0;
 383
 384                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 385                         if (iptr->line != currentline) {
 386                                 linenumbertable_list_entry_add(cd, iptr->line);
 387                                 currentline = iptr->line;
 388                         }
 389
 390                         MCODECHECK(1024);                         /* 1KB should be enough */
 391
 392                 switch (iptr->opc) {
 393                 case ICMD_NOP:        /* ...  ==> ...                                 */
 394                 case ICMD_POP:        /* ..., value  ==> ...                          */
 395                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 396                         break;
 397
 398                 case ICMD_INLINE_START:
 399
 400                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 401                         break;
 402
 403                 case ICMD_INLINE_BODY:
 404
 405                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 406                         linenumbertable_list_entry_add_inline_start(cd, iptr);
 407                         linenumbertable_list_entry_add(cd, iptr->line);
 408                         break;
 409
 410                 case ICMD_INLINE_END:
 411
 412                         linenumbertable_list_entry_add_inline_end(cd, iptr);
 413                         linenumbertable_list_entry_add(cd, iptr->line);
 414                         break;
 415
 416                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 417
 418                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 419                         emit_nullpointer_check(cd, iptr, s1);
 420                         break;
 421
 422                 /* constant operations ************************************************/
 423
 424                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 425
 426                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 427                         ICONST(d, iptr->sx.val.i);
 428                         emit_store_dst(jd, iptr, d);
 429                         break;
 430
 431                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 432
 433                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 434                         LCONST(d, iptr->sx.val.l);
 435                         emit_store_dst(jd, iptr, d);
 436                         break;
 437
 438                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 439
 440                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 441                         disp = dseg_add_float(cd, iptr->sx.val.f);
 442                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 443                         emit_store_dst(jd, iptr, d);
 444                         break;
 445
 446                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 447
 448                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 449                         disp = dseg_add_double(cd, iptr->sx.val.d);
 450                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 451                         emit_store_dst(jd, iptr, d);
 452                         break;
 453
 454                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 455
 456                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 457
 458                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 459                                 cr   = iptr->sx.val.c.ref;
 460                                 disp = dseg_add_unique_address(cd, cr);
 461
 462 /*                              PROFILE_CYCLE_STOP; */
 463
 464                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
 465                                                                           cr, disp);
 466
 467 /*                              PROFILE_CYCLE_START; */
 468
 469                                 M_ALD(d, RIP, disp);
 470                         }
 471                         else {
 472                                 if (iptr->sx.val.anyptr == 0) {
 473                                         M_CLR(d);
 474                                 }
 475                                 else {
 476                                         disp = dseg_add_address(cd, iptr->sx.val.anyptr);
 477                                         M_ALD(d, RIP, disp);
 478                                 }
 479                         }
 480                         emit_store_dst(jd, iptr, d);
 481                         break;
 482
 483
 484                 /* load/store/copy/move operations ************************************/
 485
 486                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 487                 case ICMD_ALOAD:      /* s1 = local variable                          */
 488                 case ICMD_LLOAD:
 489                 case ICMD_FLOAD:
 490                 case ICMD_DLOAD:
 491                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 492                 case ICMD_LSTORE:
 493                 case ICMD_FSTORE:
 494                 case ICMD_DSTORE:
 495                 case ICMD_COPY:
 496                 case ICMD_MOVE:
 497
 498                         emit_copy(jd, iptr);
 499                         break;
 500
 501                 case ICMD_ASTORE:
 502                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 503                                 emit_copy(jd, iptr);
 504                         break;
 505
 506                 /* integer operations *************************************************/
 507
 508                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 509
 510                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 511                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 512                         M_INTMOVE(s1, d);
 513                         M_INEG(d);
 514                         emit_store_dst(jd, iptr, d);
 515                         break;
 516
 517                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 518
 519                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 520                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 521                         M_INTMOVE(s1, d);
 522                         M_LNEG(d);
 523                         emit_store_dst(jd, iptr, d);
 524                         break;
 525
 526                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 527
 528                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 529                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 530                         M_ISEXT(s1, d);
 531                         emit_store_dst(jd, iptr, d);
 532                         break;
 533
 534                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 535
 536                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 537                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 538                         M_IMOV(s1, d);
 539                         emit_store_dst(jd, iptr, d);
 540                         break;
 541
 542                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 543
 544                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 545                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 546                         M_BSEXT(s1, d);
 547                         emit_store_dst(jd, iptr, d);
 548                         break;
 549
 550                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 551
 552                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 553                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 554                         M_CZEXT(s1, d);
 555                         emit_store_dst(jd, iptr, d);
 556                         break;
 557
 558                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 559
 560                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 561                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 562                         M_SSEXT(s1, d);
 563                         emit_store_dst(jd, iptr, d);
 564                         break;
 565
 566
 567                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 568
 569                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 570                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 571                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 572                         if (s2 == d)
 573                                 M_IADD(s1, d);
 574                         else {
 575                                 M_INTMOVE(s1, d);
 576                                 M_IADD(s2, d);
 577                         }
 578                         emit_store_dst(jd, iptr, d);
 579                         break;
 580
 581                 case ICMD_IINC:
 582                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 583                                       /* sx.val.i = constant                             */
 584
 585                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 586                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 587
 588                         /* Using inc and dec is not faster than add (tested with
 589                            sieve). */
 590
 591                         M_INTMOVE(s1, d);
 592                         M_IADD_IMM(iptr->sx.val.i, d);
 593                         emit_store_dst(jd, iptr, d);
 594                         break;
 595
 596                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 597
 598                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 599                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 600                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 601                         if (s2 == d)
 602                                 M_LADD(s1, d);
 603                         else {
 604                                 M_INTMOVE(s1, d);
 605                                 M_LADD(s2, d);
 606                         }
 607                         emit_store_dst(jd, iptr, d);
 608                         break;
 609
 610                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 611                                       /* sx.val.l = constant                             */
 612
 613                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 614                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 615                         M_INTMOVE(s1, d);
 616                         if (IS_IMM32(iptr->sx.val.l))
 617                                 M_LADD_IMM(iptr->sx.val.l, d);
 618                         else {
 619                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 620                                 M_LADD(REG_ITMP2, d);
 621                         }
 622                         emit_store_dst(jd, iptr, d);
 623                         break;
 624
 625                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 626
 627                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 628                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 629                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 630                         if (s2 == d) {
 631                                 M_INTMOVE(s1, REG_ITMP1);
 632                                 M_ISUB(s2, REG_ITMP1);
 633                                 M_INTMOVE(REG_ITMP1, d);
 634                         } else {
 635                                 M_INTMOVE(s1, d);
 636                                 M_ISUB(s2, d);
 637                         }
 638                         emit_store_dst(jd, iptr, d);
 639                         break;
 640
 641                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 642                                       /* sx.val.i = constant                             */
 643
 644                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 645                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 646                         M_INTMOVE(s1, d);
 647                         M_ISUB_IMM(iptr->sx.val.i, d);
 648                         emit_store_dst(jd, iptr, d);
 649                         break;
 650
 651                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 652
 653                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 654                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 655                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 656                         if (s2 == d) {
 657                                 M_INTMOVE(s1, REG_ITMP1);
 658                                 M_LSUB(s2, REG_ITMP1);
 659                                 M_INTMOVE(REG_ITMP1, d);
 660                         } else {
 661                                 M_INTMOVE(s1, d);
 662                                 M_LSUB(s2, d);
 663                         }
 664                         emit_store_dst(jd, iptr, d);
 665                         break;
 666
 667                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 668                                       /* sx.val.l = constant                             */
 669
 670                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 671                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 672                         M_INTMOVE(s1, d);
 673                         if (IS_IMM32(iptr->sx.val.l))
 674                                 M_LSUB_IMM(iptr->sx.val.l, d);
 675                         else {
 676                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 677                                 M_LSUB(REG_ITMP2, d);
 678                         }
 679                         emit_store_dst(jd, iptr, d);
 680                         break;
 681
 682                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 683
 684                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 685                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 686                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 687                         if (s2 == d)
 688                                 M_IMUL(s1, d);
 689                         else {
 690                                 M_INTMOVE(s1, d);
 691                                 M_IMUL(s2, d);
 692                         }
 693                         emit_store_dst(jd, iptr, d);
 694                         break;
 695
 696                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 697                                       /* sx.val.i = constant                             */
 698
 699                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 700                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 701                         if (iptr->sx.val.i == 2) {
 702                                 M_INTMOVE(s1, d);
 703                                 M_ISLL_IMM(1, d);
 704                         } else
 705                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 706                         emit_store_dst(jd, iptr, d);
 707                         break;
 708
 709                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 710
 711                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 712                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 713                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 714                         if (s2 == d)
 715                                 M_LMUL(s1, d);
 716                         else {
 717                                 M_INTMOVE(s1, d);
 718                                 M_LMUL(s2, d);
 719                         }
 720                         emit_store_dst(jd, iptr, d);
 721                         break;
 722
 723                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 724                                       /* sx.val.l = constant                             */
 725
 726                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 727                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 728                         if (IS_IMM32(iptr->sx.val.l))
 729                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 730                         else {
 731                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 732                                 M_INTMOVE(s1, d);
 733                                 M_LMUL(REG_ITMP2, d);
 734                         }
 735                         emit_store_dst(jd, iptr, d);
 736                         break;
 737
 738                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 739
 740                         s1 = emit_load_s1(jd, iptr, RAX);
 741                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 742                         d = codegen_reg_of_dst(jd, iptr, RAX);
 743
 744                         M_INTMOVE(s1, RAX);
 745                         M_INTMOVE(s2, REG_ITMP3);
 746                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 747
 748                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 749
 750                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 751                         M_BNE(4 + 6);
 752                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 753                         M_BEQ(1 + 3);                                          /* 6 bytes */
 754
 755                         emit_cltd(cd);                                         /* 1 byte  */
 756                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 757
 758                         M_INTMOVE(RAX, d);
 759                         emit_store_dst(jd, iptr, d);
 760                         dst = VAROP(iptr->dst);
 761                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 762                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 763                         break;
 764
 765                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 766
 767                         s1 = emit_load_s1(jd, iptr, RAX);
 768                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 769                         d = codegen_reg_of_dst(jd, iptr, RDX);
 770
 771                         M_INTMOVE(s1, RAX);
 772                         M_INTMOVE(s2, REG_ITMP3);
 773                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 774
 775                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 776
 777                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 778                         M_BNE(3 + 4 + 6);
 779                         M_CLR(RDX);                                            /* 3 bytes */
 780                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 781                         M_BEQ(1 + 3);                                          /* 6 bytes */
 782
 783                         emit_cltd(cd);                                         /* 1 byte  */
 784                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 785
 786                         M_INTMOVE(RDX, d);
 787                         emit_store_dst(jd, iptr, d);
 788                         dst = VAROP(iptr->dst);
 789                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 790                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 791                         break;
 792
 793                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 794                                       /* sx.val.i = constant                             */
 795
 796                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 797                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 798                         M_INTMOVE(s1, REG_ITMP1);
 799                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 800                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 801                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 802                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 803                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 804                         emit_store_dst(jd, iptr, d);
 805                         break;
 806
 807                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 808                                       /* sx.val.i = constant                             */
 809
 810                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 811                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 812                         M_INTMOVE(s1, REG_ITMP1);
 813                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 814                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 815                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 816                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 817                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 818                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 819                         emit_store_dst(jd, iptr, d);
 820                         break;
 821
 822
 823                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 824
 825                         s1 = emit_load_s1(jd, iptr, RAX);
 826                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 827                         d = codegen_reg_of_dst(jd, iptr, RAX);
 828
 829                         M_INTMOVE(s1, RAX);
 830                         M_INTMOVE(s2, REG_ITMP3);
 831                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 832
 833                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 834
 835                         /* check as described in jvm spec */
 836                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 837                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 838                         M_BNE(4 + 6);
 839                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 840                         M_BEQ(2 + 3);                                          /* 6 bytes */
 841
 842                         emit_cqto(cd);                                         /* 2 bytes */
 843                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 844
 845                         M_INTMOVE(RAX, d);
 846                         emit_store_dst(jd, iptr, d);
 847                         dst = VAROP(iptr->dst);
 848                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 849                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 850                         break;
 851
 852                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 853
 854                         s1 = emit_load_s1(jd, iptr, RAX);
 855                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 856                         d = codegen_reg_of_dst(jd, iptr, RDX);
 857
 858                         M_INTMOVE(s1, RAX);
 859                         M_INTMOVE(s2, REG_ITMP3);
 860                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 861
 862                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 863
 864                         /* check as described in jvm spec */
 865                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 866                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 867                         M_BNE(3 + 4 + 6);
 868                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 869                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 870                         M_BEQ(2 + 3);                                          /* 6 bytes */
 871
 872                         emit_cqto(cd);                                         /* 2 bytes */
 873                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 874
 875                         M_INTMOVE(RDX, d);
 876                         emit_store_dst(jd, iptr, d);
 877                         dst = VAROP(iptr->dst);
 878                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 879                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 880                         break;
 881
 882                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 883                                       /* sx.val.i = constant                             */
 884
 885                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 886                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 887                         M_INTMOVE(s1, REG_ITMP1);
 888                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 889                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 890                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 891                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 892                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 893                         emit_store_dst(jd, iptr, d);
 894                         break;
 895
 896                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 897                                       /* sx.val.l = constant                             */
 898
 899                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 900                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 901                         M_INTMOVE(s1, REG_ITMP1);
 902                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 903                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 904                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 905                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 906                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 907                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 908                         emit_store_dst(jd, iptr, d);
 909                         break;
 910
 911                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 912
 913                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 914                         emit_ishift(jd, SHIFT_SHL, iptr);
 915                         break;
 916
 917                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 918                                       /* sx.val.i = constant                             */
 919
 920                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 921                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 922                         M_INTMOVE(s1, d);
 923                         M_ISLL_IMM(iptr->sx.val.i, d);
 924                         emit_store_dst(jd, iptr, d);
 925                         break;
 926
 927                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 928
 929                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 930                         emit_ishift(jd, SHIFT_SAR, iptr);
 931                         break;
 932
 933                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 934                                       /* sx.val.i = constant                             */
 935
 936                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 937                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 938                         M_INTMOVE(s1, d);
 939                         M_ISRA_IMM(iptr->sx.val.i, d);
 940                         emit_store_dst(jd, iptr, d);
 941                         break;
 942
 943                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 944
 945                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 946                         emit_ishift(jd, SHIFT_SHR, iptr);
 947                         break;
 948
 949                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 950                                       /* sx.val.i = constant                             */
 951
 952                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 953                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 954                         M_INTMOVE(s1, d);
 955                         M_ISRL_IMM(iptr->sx.val.i, d);
 956                         emit_store_dst(jd, iptr, d);
 957                         break;
 958
 959                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 960
 961                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 962                         emit_lshift(jd, SHIFT_SHL, iptr);
 963                         break;
 964
 965         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 966                                           /* sx.val.i = constant                             */
 967
 968                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 969                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 970                         M_INTMOVE(s1, d);
 971                         M_LSLL_IMM(iptr->sx.val.i, d);
 972                         emit_store_dst(jd, iptr, d);
 973                         break;
 974
 975                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 976
 977                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 978                         emit_lshift(jd, SHIFT_SAR, iptr);
 979                         break;
 980
 981                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 982                                       /* sx.val.i = constant                             */
 983
 984                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 985                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 986                         M_INTMOVE(s1, d);
 987                         M_LSRA_IMM(iptr->sx.val.i, d);
 988                         emit_store_dst(jd, iptr, d);
 989                         break;
 990
 991                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 992
 993                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 994                         emit_lshift(jd, SHIFT_SHR, iptr);
 995                         break;
 996
 997                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 998                                       /* sx.val.l = constant                             */
 999
1000                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1001                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1002                         M_INTMOVE(s1, d);
1003                         M_LSRL_IMM(iptr->sx.val.i, d);
1004                         emit_store_dst(jd, iptr, d);
1005                         break;
1006
1007                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1008
1009                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1010                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1011                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1012                         if (s2 == d)
1013                                 M_IAND(s1, d);
1014                         else {
1015                                 M_INTMOVE(s1, d);
1016                                 M_IAND(s2, d);
1017                         }
1018                         emit_store_dst(jd, iptr, d);
1019                         break;
1020
1021                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1022                                       /* sx.val.i = constant                             */
1023
1024                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1025                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1026                         M_INTMOVE(s1, d);
1027                         M_IAND_IMM(iptr->sx.val.i, d);
1028                         emit_store_dst(jd, iptr, d);
1029                         break;
1030
1031                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1032
1033                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1034                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1035                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1036                         if (s2 == d)
1037                                 M_LAND(s1, d);
1038                         else {
1039                                 M_INTMOVE(s1, d);
1040                                 M_LAND(s2, d);
1041                         }
1042                         emit_store_dst(jd, iptr, d);
1043                         break;
1044
1045                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1046                                       /* sx.val.l = constant                             */
1047
1048                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1049                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1050                         M_INTMOVE(s1, d);
1051                         if (IS_IMM32(iptr->sx.val.l))
1052                                 M_LAND_IMM(iptr->sx.val.l, d);
1053                         else {
1054                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1055                                 M_LAND(REG_ITMP2, d);
1056                         }
1057                         emit_store_dst(jd, iptr, d);
1058                         break;
1059
1060                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1061
1062                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1063                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1064                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1065                         if (s2 == d)
1066                                 M_IOR(s1, d);
1067                         else {
1068                                 M_INTMOVE(s1, d);
1069                                 M_IOR(s2, d);
1070                         }
1071                         emit_store_dst(jd, iptr, d);
1072                         break;
1073
1074                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1075                                       /* sx.val.i = constant                             */
1076
1077                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1078                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1079                         M_INTMOVE(s1, d);
1080                         M_IOR_IMM(iptr->sx.val.i, d);
1081                         emit_store_dst(jd, iptr, d);
1082                         break;
1083
1084                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1085
1086                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1087                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1088                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1089                         if (s2 == d)
1090                                 M_LOR(s1, d);
1091                         else {
1092                                 M_INTMOVE(s1, d);
1093                                 M_LOR(s2, d);
1094                         }
1095                         emit_store_dst(jd, iptr, d);
1096                         break;
1097
1098                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1099                                       /* sx.val.l = constant                             */
1100
1101                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1102                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1103                         M_INTMOVE(s1, d);
1104                         if (IS_IMM32(iptr->sx.val.l))
1105                                 M_LOR_IMM(iptr->sx.val.l, d);
1106                         else {
1107                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1108                                 M_LOR(REG_ITMP2, d);
1109                         }
1110                         emit_store_dst(jd, iptr, d);
1111                         break;
1112
1113                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1114
1115                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1116                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1117                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1118                         if (s2 == d)
1119                                 M_IXOR(s1, d);
1120                         else {
1121                                 M_INTMOVE(s1, d);
1122                                 M_IXOR(s2, d);
1123                         }
1124                         emit_store_dst(jd, iptr, d);
1125                         break;
1126
1127                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1128                                       /* sx.val.i = constant                             */
1129
1130                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1131                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1132                         M_INTMOVE(s1, d);
1133                         M_IXOR_IMM(iptr->sx.val.i, d);
1134                         emit_store_dst(jd, iptr, d);
1135                         break;
1136
1137                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1138
1139                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1140                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1141                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1142                         if (s2 == d)
1143                                 M_LXOR(s1, d);
1144                         else {
1145                                 M_INTMOVE(s1, d);
1146                                 M_LXOR(s2, d);
1147                         }
1148                         emit_store_dst(jd, iptr, d);
1149                         break;
1150
1151                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1152                                       /* sx.val.l = constant                             */
1153
1154                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1155                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1156                         M_INTMOVE(s1, d);
1157                         if (IS_IMM32(iptr->sx.val.l))
1158                                 M_LXOR_IMM(iptr->sx.val.l, d);
1159                         else {
1160                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1161                                 M_LXOR(REG_ITMP2, d);
1162                         }
1163                         emit_store_dst(jd, iptr, d);
1164                         break;
1165
1166
1167                 /* floating operations ************************************************/
1168
1169                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1170
1171                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1172                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1173                         disp = dseg_add_s4(cd, 0x80000000);
1174                         M_FLTMOVE(s1, d);
1175                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1176                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1177                         emit_store_dst(jd, iptr, d);
1178                         break;
1179
1180                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1181
1182                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1183                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1184                         disp = dseg_add_s8(cd, 0x8000000000000000);
1185                         M_FLTMOVE(s1, d);
1186                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1187                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1188                         emit_store_dst(jd, iptr, d);
1189                         break;
1190
1191                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1192
1193                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1194                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1195                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1196                         if (s2 == d)
1197                                 M_FADD(s1, d);
1198                         else {
1199                                 M_FLTMOVE(s1, d);
1200                                 M_FADD(s2, d);
1201                         }
1202                         emit_store_dst(jd, iptr, d);
1203                         break;
1204
1205                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1206
1207                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1208                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1209                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1210                         if (s2 == d)
1211                                 M_DADD(s1, d);
1212                         else {
1213                                 M_FLTMOVE(s1, d);
1214                                 M_DADD(s2, d);
1215                         }
1216                         emit_store_dst(jd, iptr, d);
1217                         break;
1218
1219                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1220
1221                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1222                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1223                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1224                         if (s2 == d) {
1225                                 M_FLTMOVE(s2, REG_FTMP2);
1226                                 s2 = REG_FTMP2;
1227                         }
1228                         M_FLTMOVE(s1, d);
1229                         M_FSUB(s2, d);
1230                         emit_store_dst(jd, iptr, d);
1231                         break;
1232
1233                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1234
1235                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1236                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1237                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1238                         if (s2 == d) {
1239                                 M_FLTMOVE(s2, REG_FTMP2);
1240                                 s2 = REG_FTMP2;
1241                         }
1242                         M_FLTMOVE(s1, d);
1243                         M_DSUB(s2, d);
1244                         emit_store_dst(jd, iptr, d);
1245                         break;
1246
1247                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1248
1249                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1250                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1251                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1252                         if (s2 == d)
1253                                 M_FMUL(s1, d);
1254                         else {
1255                                 M_FLTMOVE(s1, d);
1256                                 M_FMUL(s2, d);
1257                         }
1258                         emit_store_dst(jd, iptr, d);
1259                         break;
1260
1261                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1262
1263                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1264                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1265                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1266                         if (s2 == d)
1267                                 M_DMUL(s1, d);
1268                         else {
1269                                 M_FLTMOVE(s1, d);
1270                                 M_DMUL(s2, d);
1271                         }
1272                         emit_store_dst(jd, iptr, d);
1273                         break;
1274
1275                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1276
1277                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1278                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1279                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1280                         if (s2 == d) {
1281                                 M_FLTMOVE(s2, REG_FTMP2);
1282                                 s2 = REG_FTMP2;
1283                         }
1284                         M_FLTMOVE(s1, d);
1285                         M_FDIV(s2, d);
1286                         emit_store_dst(jd, iptr, d);
1287                         break;
1288
1289                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1290
1291                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1292                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1293                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1294                         if (s2 == d) {
1295                                 M_FLTMOVE(s2, REG_FTMP2);
1296                                 s2 = REG_FTMP2;
1297                         }
1298                         M_FLTMOVE(s1, d);
1299                         M_DDIV(s2, d);
1300                         emit_store_dst(jd, iptr, d);
1301                         break;
1302
1303                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1304
1305                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1306                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1307                         M_CVTIF(s1, d);
1308                         emit_store_dst(jd, iptr, d);
1309                         break;
1310
1311                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1312
1313                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1314                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1315                         M_CVTID(s1, d);
1316                         emit_store_dst(jd, iptr, d);
1317                         break;
1318
1319                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1320
1321                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1322                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1323                         M_CVTLF(s1, d);
1324                         emit_store_dst(jd, iptr, d);
1325                         break;
1326
1327                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1328
1329                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1330                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1331                         M_CVTLD(s1, d);
1332                         emit_store_dst(jd, iptr, d);
1333                         break;
1334
1335                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1336
1337                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1338                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1339                         M_CVTFI(s1, d);
1340                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1341                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1342                                 ((REG_RESULT == d) ? 0 : 3);
1343                         M_BNE(disp);
1344                         M_FLTMOVE(s1, REG_FTMP1);
1345                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1346                         M_CALL(REG_ITMP2);
1347                         M_INTMOVE(REG_RESULT, d);
1348                         emit_store_dst(jd, iptr, d);
1349                         break;
1350
1351                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1352
1353                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1354                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1355                         M_CVTDI(s1, d);
1356                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1357                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1358                                 ((REG_RESULT == d) ? 0 : 3);
1359                         M_BNE(disp);
1360                         M_FLTMOVE(s1, REG_FTMP1);
1361                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1362                         M_CALL(REG_ITMP2);
1363                         M_INTMOVE(REG_RESULT, d);
1364                         emit_store_dst(jd, iptr, d);
1365                         break;
1366
1367                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1368
1369                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1370                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1371                         M_CVTFL(s1, d);
1372                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1373                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1374                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1375                                 ((REG_RESULT == d) ? 0 : 3);
1376                         M_BNE(disp);
1377                         M_FLTMOVE(s1, REG_FTMP1);
1378                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1379                         M_CALL(REG_ITMP2);
1380                         M_INTMOVE(REG_RESULT, d);
1381                         emit_store_dst(jd, iptr, d);
1382                         break;
1383
1384                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1385
1386                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1387                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1388                         M_CVTDL(s1, d);
1389                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1390                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1391                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1392                                 ((REG_RESULT == d) ? 0 : 3);
1393                         M_BNE(disp);
1394                         M_FLTMOVE(s1, REG_FTMP1);
1395                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1396                         M_CALL(REG_ITMP2);
1397                         M_INTMOVE(REG_RESULT, d);
1398                         emit_store_dst(jd, iptr, d);
1399                         break;
1400
1401                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1402
1403                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1404                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1405                         M_CVTFD(s1, d);
1406                         emit_store_dst(jd, iptr, d);
1407                         break;
1408
1409                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1410
1411                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1412                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1413                         M_CVTDF(s1, d);
1414                         emit_store_dst(jd, iptr, d);
1415                         break;
1416
1417                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1418                                           /* == => 0, < => 1, > => -1 */
1419
1420                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1421                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1422                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1423                         M_CLR(d);
1424                         M_MOV_IMM(1, REG_ITMP1);
1425                         M_MOV_IMM(-1, REG_ITMP2);
1426                         emit_ucomiss_reg_reg(cd, s1, s2);
1427                         M_CMOVULT(REG_ITMP1, d);
1428                         M_CMOVUGT(REG_ITMP2, d);
1429                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1430                         emit_store_dst(jd, iptr, d);
1431                         break;
1432
1433                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1434                                           /* == => 0, < => 1, > => -1 */
1435
1436                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1437                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1438                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1439                         M_CLR(d);
1440                         M_MOV_IMM(1, REG_ITMP1);
1441                         M_MOV_IMM(-1, REG_ITMP2);
1442                         emit_ucomiss_reg_reg(cd, s1, s2);
1443                         M_CMOVULT(REG_ITMP1, d);
1444                         M_CMOVUGT(REG_ITMP2, d);
1445                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1446                         emit_store_dst(jd, iptr, d);
1447                         break;
1448
1449                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1450                                           /* == => 0, < => 1, > => -1 */
1451
1452                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1453                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1454                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1455                         M_CLR(d);
1456                         M_MOV_IMM(1, REG_ITMP1);
1457                         M_MOV_IMM(-1, REG_ITMP2);
1458                         emit_ucomisd_reg_reg(cd, s1, s2);
1459                         M_CMOVULT(REG_ITMP1, d);
1460                         M_CMOVUGT(REG_ITMP2, d);
1461                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1462                         emit_store_dst(jd, iptr, d);
1463                         break;
1464
1465                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1466                                           /* == => 0, < => 1, > => -1 */
1467
1468                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1469                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1470                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1471                         M_CLR(d);
1472                         M_MOV_IMM(1, REG_ITMP1);
1473                         M_MOV_IMM(-1, REG_ITMP2);
1474                         emit_ucomisd_reg_reg(cd, s1, s2);
1475                         M_CMOVULT(REG_ITMP1, d);
1476                         M_CMOVUGT(REG_ITMP2, d);
1477                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1478                         emit_store_dst(jd, iptr, d);
1479                         break;
1480
1481
1482                 /* memory operations **************************************************/
1483
1484                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1485
1486                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1487                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1488                         /* implicit null-pointer check */
1489                         M_ILD(d, s1, OFFSET(java_array_t, size));
1490                         emit_store_dst(jd, iptr, d);
1491                         break;
1492
1493                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1494
1495                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1496                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1497                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1498                         /* implicit null-pointer check */
1499                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1500                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1501                         emit_store_dst(jd, iptr, d);
1502                         break;
1503
1504                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1505
1506                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1507                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1508                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1509                         /* implicit null-pointer check */
1510                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1511                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1512                         emit_store_dst(jd, iptr, d);
1513                         break;
1514
1515                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1516
1517                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1518                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1519                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1520                         /* implicit null-pointer check */
1521                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1522                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1523                         emit_store_dst(jd, iptr, d);
1524                         break;
1525
1526                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1527
1528                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1529                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1530                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1531                         /* implicit null-pointer check */
1532                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1533                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1534                         emit_store_dst(jd, iptr, d);
1535                         break;
1536
1537                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1538
1539                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1540                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1541                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1542                         /* implicit null-pointer check */
1543                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1544                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1545                         emit_store_dst(jd, iptr, d);
1546                         break;
1547
1548                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1549
1550                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1551                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1552                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1553                         /* implicit null-pointer check */
1554                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1555                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1556                         emit_store_dst(jd, iptr, d);
1557                         break;
1558
1559                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1560
1561                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1562                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1563                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1564                         /* implicit null-pointer check */
1565                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1566                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1567                         emit_store_dst(jd, iptr, d);
1568                         break;
1569
1570                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1571
1572                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1573                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1574                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1575                         /* implicit null-pointer check */
1576                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1577                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1578                         emit_store_dst(jd, iptr, d);
1579                         break;
1580
1581
1582                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1583
1584                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1585                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1586                         /* implicit null-pointer check */
1587                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1588                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1589                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1590                         break;
1591
1592                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1593
1594                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1595                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1596                         /* implicit null-pointer check */
1597                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1598                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1599                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1600                         break;
1601
1602                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1603
1604                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1605                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1606                         /* implicit null-pointer check */
1607                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1608                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1609                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1610                         break;
1611
1612                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1613
1614                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1615                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1616                         /* implicit null-pointer check */
1617                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1618                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1619                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1620                         break;
1621
1622                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1623
1624                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1625                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1626                         /* implicit null-pointer check */
1627                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1628                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1629                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1630                         break;
1631
1632                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1633
1634                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1635                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1636                         /* implicit null-pointer check */
1637                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1638                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1639                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1640                         break;
1641
1642                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1643
1644                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1645                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1646                         /* implicit null-pointer check */
1647                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1648                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1649                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1650                         break;
1651
1652                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1653
1654                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1655                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1656                         /* implicit null-pointer check */
1657                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1658                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1659
1660                         M_MOV(s1, REG_A0);
1661                         M_MOV(s3, REG_A1);
1662                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
1663                         M_CALL(REG_ITMP1);
1664                         emit_arraystore_check(cd, iptr);
1665
1666                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1667                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1668                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1669                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1670                         break;
1671
1672
1673                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1674
1675                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1676                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1677                         /* implicit null-pointer check */
1678                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1679                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1680                         break;
1681
1682                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1683
1684                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1685                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1686                         /* implicit null-pointer check */
1687                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1688                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1689                         break;
1690
1691                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1692
1693                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1694                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1695                         /* implicit null-pointer check */
1696                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1697                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1698                         break;
1699
1700                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1701
1702                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1703                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1704                         /* implicit null-pointer check */
1705                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1706                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1707                         break;
1708
1709                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1710
1711                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1712                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1713                         /* implicit null-pointer check */
1714                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1715
1716                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1717                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1718                         }
1719                         else {
1720                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1721                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1722                         }
1723                         break;
1724
1725                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1726
1727                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1728                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1729                         /* implicit null-pointer check */
1730                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1731                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1732                         break;
1733
1734
1735                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1736
1737                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1738                                 uf        = iptr->sx.s23.s3.uf;
1739                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1740                                 disp      = dseg_add_unique_address(cd, uf);
1741
1742 /*                              PROFILE_CYCLE_STOP; */
1743
1744                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1745
1746 /*                              PROFILE_CYCLE_START; */
1747                         }
1748                         else {
1749                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1750                                 fieldtype = fi->type;
1751                                 disp      = dseg_add_address(cd, fi->value);
1752
1753                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1754                                         PROFILE_CYCLE_STOP;
1755
1756                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1757                                                                                   fi->class, 0);
1758
1759                                         PROFILE_CYCLE_START;
1760                                 }
1761                         }
1762
1763                         /* This approach is much faster than moving the field
1764                            address inline into a register. */
1765
1766                         M_ALD(REG_ITMP1, RIP, disp);
1767
1768                         switch (fieldtype) {
1769                         case TYPE_INT:
1770                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1771                                 M_ILD(d, REG_ITMP1, 0);
1772                                 break;
1773                         case TYPE_LNG:
1774                         case TYPE_ADR:
1775                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1776                                 M_LLD(d, REG_ITMP1, 0);
1777                                 break;
1778                         case TYPE_FLT:
1779                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1780                                 M_FLD(d, REG_ITMP1, 0);
1781                                 break;
1782                         case TYPE_DBL:
1783                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1784                                 M_DLD(d, REG_ITMP1, 0);
1785                                 break;
1786                         }
1787                         emit_store_dst(jd, iptr, d);
1788                         break;
1789
1790                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1791
1792                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1793                                 uf        = iptr->sx.s23.s3.uf;
1794                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1795                                 disp      = dseg_add_unique_address(cd, uf);
1796
1797 /*                              PROFILE_CYCLE_STOP; */
1798
1799                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1800
1801 /*                              PROFILE_CYCLE_START; */
1802                         }
1803                         else {
1804                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1805                                 fieldtype = fi->type;
1806                                 disp      = dseg_add_address(cd, fi->value);
1807
1808                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1809                                         PROFILE_CYCLE_STOP;
1810
1811                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1812                                                                                   fi->class, 0);
1813
1814                                         PROFILE_CYCLE_START;
1815                                 }
1816                         }
1817
1818                         /* This approach is much faster than moving the field
1819                            address inline into a register. */
1820
1821                         M_ALD(REG_ITMP1, RIP, disp);
1822
1823                         switch (fieldtype) {
1824                         case TYPE_INT:
1825                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1826                                 M_IST(s1, REG_ITMP1, 0);
1827                                 break;
1828                         case TYPE_LNG:
1829                         case TYPE_ADR:
1830                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1831                                 M_LST(s1, REG_ITMP1, 0);
1832                                 break;
1833                         case TYPE_FLT:
1834                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1835                                 M_FST(s1, REG_ITMP1, 0);
1836                                 break;
1837                         case TYPE_DBL:
1838                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1839                                 M_DST(s1, REG_ITMP1, 0);
1840                                 break;
1841                         }
1842                         break;
1843
1844                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1845                                           /* val = value (in current instruction)     */
1846                                           /* following NOP)                           */
1847
1848                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1849                                 uf        = iptr->sx.s23.s3.uf;
1850                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1851                                 disp      = dseg_add_unique_address(cd, uf);
1852
1853 /*                              PROFILE_CYCLE_STOP; */
1854
1855                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1856
1857 /*                              PROFILE_CYCLE_START; */
1858                         }
1859                         else {
1860                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1861                                 fieldtype = fi->type;
1862                                 disp      = dseg_add_address(cd, fi->value);
1863
1864                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1865                                         PROFILE_CYCLE_STOP;
1866
1867                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1868                                                                                   fi->class, 0);
1869
1870                                         PROFILE_CYCLE_START;
1871                                 }
1872                         }
1873
1874                         /* This approach is much faster than moving the field
1875                            address inline into a register. */
1876
1877                         M_ALD(REG_ITMP1, RIP, disp);
1878
1879                         switch (fieldtype) {
1880                         case TYPE_INT:
1881                         case TYPE_FLT:
1882                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1883                                 break;
1884                         case TYPE_LNG:
1885                         case TYPE_ADR:
1886                         case TYPE_DBL:
1887                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1888                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1889                                 else {
1890                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1891                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1892                                 }
1893                                 break;
1894                         }
1895                         break;
1896
1897                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1898
1899                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1900
1901                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1902                                 uf        = iptr->sx.s23.s3.uf;
1903                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1904                                 disp      = 0;
1905
1906 /*                              PROFILE_CYCLE_STOP; */
1907
1908                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1909
1910 /*                              PROFILE_CYCLE_START; */
1911                         }
1912                         else {
1913                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1914                                 fieldtype = fi->type;
1915                                 disp      = fi->offset;
1916                         }
1917
1918                         /* implicit null-pointer check */
1919                         switch (fieldtype) {
1920                         case TYPE_INT:
1921                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1922                                 M_ILD32(d, s1, disp);
1923                                 break;
1924                         case TYPE_LNG:
1925                         case TYPE_ADR:
1926                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1927                                 M_LLD32(d, s1, disp);
1928                                 break;
1929                         case TYPE_FLT:
1930                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1931                                 M_FLD32(d, s1, disp);
1932                                 break;
1933                         case TYPE_DBL:
1934                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1935                                 M_DLD32(d, s1, disp);
1936                                 break;
1937                         }
1938                         emit_store_dst(jd, iptr, d);
1939                         break;
1940
1941                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1942
1943                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1944                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1945
1946                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1947                                 uf        = iptr->sx.s23.s3.uf;
1948                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1949                                 disp      = 0;
1950
1951 /*                              PROFILE_CYCLE_STOP; */
1952
1953                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1954
1955 /*                              PROFILE_CYCLE_START; */
1956                         }
1957                         else {
1958                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1959                                 fieldtype = fi->type;
1960                                 disp      = fi->offset;
1961                         }
1962
1963                         /* implicit null-pointer check */
1964                         switch (fieldtype) {
1965                         case TYPE_INT:
1966                                 M_IST32(s2, s1, disp);
1967                                 break;
1968                         case TYPE_LNG:
1969                         case TYPE_ADR:
1970                                 M_LST32(s2, s1, disp);
1971                                 break;
1972                         case TYPE_FLT:
1973                                 M_FST32(s2, s1, disp);
1974                                 break;
1975                         case TYPE_DBL:
1976                                 M_DST32(s2, s1, disp);
1977                                 break;
1978                         }
1979                         break;
1980
1981                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
1982                                           /* val = value (in current instruction)     */
1983                                           /* following NOP)                           */
1984
1985                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1986
1987                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1988                                 uf        = iptr->sx.s23.s3.uf;
1989                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1990                                 disp      = 0;
1991
1992 /*                              PROFILE_CYCLE_STOP; */
1993
1994                                 patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
1995
1996 /*                              PROFILE_CYCLE_START; */
1997                         }
1998                         else {
1999                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2000                                 fieldtype = fi->type;
2001                                 disp      = fi->offset;
2002                         }
2003
2004                         /* implicit null-pointer check */
2005                         switch (fieldtype) {
2006                         case TYPE_INT:
2007                         case TYPE_FLT:
2008                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2009                                 break;
2010                         case TYPE_LNG:
2011                         case TYPE_ADR:
2012                         case TYPE_DBL:
2013                                 /* XXX why no check for IS_IMM32? */
2014                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2015                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2016                                 break;
2017                         }
2018                         break;
2019
2020
2021                 /* branch operations **************************************************/
2022
2023                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2024
2025                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2026                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2027
2028                         PROFILE_CYCLE_STOP;
2029
2030 #ifdef ENABLE_VERIFIER
2031                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2032                                 uc = iptr->sx.s23.s2.uc;
2033
2034                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2035                         }
2036 #endif /* ENABLE_VERIFIER */
2037
2038                         M_CALL_IMM(0);                            /* passing exception pc */
2039                         M_POP(REG_ITMP2_XPC);
2040
2041                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2042                         M_JMP(REG_ITMP3);
2043                         break;
2044
2045                 case ICMD_GOTO:         /* ... ==> ...                                */
2046                 case ICMD_RET:
2047
2048                         emit_br(cd, iptr->dst.block);
2049                         ALIGNCODENOP;
2050                         break;
2051
2052                 case ICMD_JSR:          /* ... ==> ...                                */
2053
2054                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2055                         ALIGNCODENOP;
2056                         break;
2057
2058                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2059                 case ICMD_IFNONNULL:
2060
2061                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2062                         M_TEST(s1);
2063                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2064                         break;
2065
2066                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2067                 case ICMD_IFLT:
2068                 case ICMD_IFLE:
2069                 case ICMD_IFNE:
2070                 case ICMD_IFGT:
2071                 case ICMD_IFGE:
2072
2073                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2074                         M_ICMP_IMM(iptr->sx.val.i, s1);
2075                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2076                         break;
2077
2078                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2079                 case ICMD_IF_LNE:
2080                 case ICMD_IF_LLT:
2081                 case ICMD_IF_LGE:
2082                 case ICMD_IF_LGT:
2083                 case ICMD_IF_LLE:
2084
2085                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2086                         if (IS_IMM32(iptr->sx.val.l))
2087                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2088                         else {
2089                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2090                                 M_LCMP(REG_ITMP2, s1);
2091                         }
2092                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2093                         break;
2094
2095                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2096                 case ICMD_IF_ICMPNE:
2097                 case ICMD_IF_ICMPLT:
2098                 case ICMD_IF_ICMPGE:
2099                 case ICMD_IF_ICMPGT:
2100                 case ICMD_IF_ICMPLE:
2101
2102                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2103                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2104                         M_ICMP(s2, s1);
2105                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2106                         break;
2107
2108                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2109                 case ICMD_IF_ACMPNE:
2110
2111                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2112                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2113                         M_LCMP(s2, s1);
2114                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2115                         break;
2116
2117                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2118                 case ICMD_IF_LCMPNE:
2119                 case ICMD_IF_LCMPLT:
2120                 case ICMD_IF_LCMPGE:
2121                 case ICMD_IF_LCMPGT:
2122                 case ICMD_IF_LCMPLE:
2123
2124                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2125                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2126                         M_LCMP(s2, s1);
2127                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2128                         break;
2129
2130                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2131                 case ICMD_LRETURN:
2132
2133                         REPLACEMENT_POINT_RETURN(cd, iptr);
2134                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2135                         M_INTMOVE(s1, REG_RESULT);
2136                         goto nowperformreturn;
2137
2138                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2139
2140                         REPLACEMENT_POINT_RETURN(cd, iptr);
2141                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2142                         M_INTMOVE(s1, REG_RESULT);
2143
2144 #ifdef ENABLE_VERIFIER
2145                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2146                                 uc = iptr->sx.s23.s2.uc;
2147
2148                                 PROFILE_CYCLE_STOP;
2149
2150                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2151
2152                                 PROFILE_CYCLE_START;
2153                         }
2154 #endif /* ENABLE_VERIFIER */
2155                         goto nowperformreturn;
2156
2157                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2158                 case ICMD_DRETURN:
2159
2160                         REPLACEMENT_POINT_RETURN(cd, iptr);
2161                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2162                         M_FLTMOVE(s1, REG_FRESULT);
2163                         goto nowperformreturn;
2164
2165                 case ICMD_RETURN:      /* ...  ==> ...                                */
2166
2167                         REPLACEMENT_POINT_RETURN(cd, iptr);
2168
2169 nowperformreturn:
2170                         {
2171                         s4 i, p;
2172
2173                         p = cd->stackframesize;
2174
2175 #if !defined(NDEBUG)
2176                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2177                                 emit_verbosecall_exit(jd);
2178 #endif /* !defined(NDEBUG) */
2179
2180 #if defined(ENABLE_THREADS)
2181                         if (checksync && code_is_synchronized(code)) {
2182                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2183
2184                                 /* we need to save the proper return value */
2185                                 switch (iptr->opc) {
2186                                 case ICMD_IRETURN:
2187                                 case ICMD_ARETURN:
2188                                 case ICMD_LRETURN:
2189                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2190                                         break;
2191                                 case ICMD_FRETURN:
2192                                 case ICMD_DRETURN:
2193                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2194                                         break;
2195                                 }
2196
2197                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2198                                 M_CALL(REG_ITMP1);
2199
2200                                 /* and now restore the proper return value */
2201                                 switch (iptr->opc) {
2202                                 case ICMD_IRETURN:
2203                                 case ICMD_ARETURN:
2204                                 case ICMD_LRETURN:
2205                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2206                                         break;
2207                                 case ICMD_FRETURN:
2208                                 case ICMD_DRETURN:
2209                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2210                                         break;
2211                                 }
2212                         }
2213 #endif
2214
2215                         /* restore saved registers */
2216
2217                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2218                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2219                         }
2220                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2221                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2222                         }
2223
2224                         /* deallocate stack */
2225
2226                         if (cd->stackframesize)
2227                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2228
2229                         /* generate method profiling code */
2230
2231                         PROFILE_CYCLE_STOP;
2232
2233                         M_RET;
2234                         }
2235                         break;
2236
2237
2238                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2239                         {
2240                                 s4 i, l;
2241                                 branch_target_t *table;
2242
2243                                 table = iptr->dst.table;
2244
2245                                 l = iptr->sx.s23.s2.tablelow;
2246                                 i = iptr->sx.s23.s3.tablehigh;
2247
2248                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2249                                 M_INTMOVE(s1, REG_ITMP1);
2250
2251                                 if (l != 0)
2252                                         M_ISUB_IMM(l, REG_ITMP1);
2253
2254                                 /* number of targets */
2255                                 i = i - l + 1;
2256
2257                 /* range check */
2258
2259                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2260                                 emit_bugt(cd, table[0].block);
2261
2262                                 /* build jump table top down and use address of lowest entry */
2263
2264                                 table += i;
2265
2266                                 while (--i >= 0) {
2267                                         dseg_add_target(cd, table->block);
2268                                         --table;
2269                                 }
2270
2271                                 /* length of dataseg after last dseg_add_target is used
2272                                    by load */
2273
2274                                 M_MOV_IMM(0, REG_ITMP2);
2275                                 dseg_adddata(cd);
2276                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2277                                 M_JMP(REG_ITMP1);
2278                         }
2279                         break;
2280
2281
2282                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2283                         {
2284                                 s4 i;
2285                                 lookup_target_t *lookup;
2286
2287                                 lookup = iptr->dst.lookup;
2288
2289                                 i = iptr->sx.s23.s2.lookupcount;
2290
2291                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2292                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2293
2294                                 while (--i >= 0) {
2295                                         M_ICMP_IMM(lookup->value, s1);
2296                                         emit_beq(cd, lookup->target.block);
2297                                         lookup++;
2298                                 }
2299
2300                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2301                                 ALIGNCODENOP;
2302                         }
2303                         break;
2304
2305
2306                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2307
2308                         REPLACEMENT_POINT_FORGC_BUILTIN(cd, iptr);
2309
2310                         bte = iptr->sx.s23.s3.bte;
2311                         md  = bte->md;
2312                         goto gen_method;
2313
2314                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2315
2316                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2317                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2318                 case ICMD_INVOKEINTERFACE:
2319
2320                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2321
2322                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2323                                 lm = NULL;
2324                                 um = iptr->sx.s23.s3.um;
2325                                 md = um->methodref->parseddesc.md;
2326                         }
2327                         else {
2328                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2329                                 um = NULL;
2330                                 md = lm->parseddesc;
2331                         }
2332
2333 gen_method:
2334                         s3 = md->paramcount;
2335
2336                         MCODECHECK((20 * s3) + 128);
2337
2338                         /* copy arguments to registers or stack location */
2339
2340                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2341                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2342                                 d   = md->params[s3].regoff;
2343
2344                                 /* already preallocated (ARGVAR)? */
2345
2346                                 if (var->flags & PREALLOC)
2347                                         continue;
2348
2349                                 if (IS_INT_LNG_TYPE(var->type)) {
2350                                         if (!md->params[s3].inmemory) {
2351                                                 s1 = emit_load(jd, iptr, var, d);
2352                                                 M_INTMOVE(s1, d);
2353                                         }
2354                                         else {
2355                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2356                                                 M_LST(s1, REG_SP, d);
2357                                         }
2358                                 }
2359                                 else {
2360                                         if (!md->params[s3].inmemory) {
2361                                                 s1 = emit_load(jd, iptr, var, d);
2362                                                 M_FLTMOVE(s1, d);
2363                                         }
2364                                         else {
2365                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2366
2367                                                 if (IS_2_WORD_TYPE(var->type))
2368                                                         M_DST(s1, REG_SP, d);
2369                                                 else
2370                                                         M_FST(s1, REG_SP, d);
2371                                         }
2372                                 }
2373                         }
2374
2375                         /* generate method profiling code */
2376
2377                         PROFILE_CYCLE_STOP;
2378
2379                         switch (iptr->opc) {
2380                         case ICMD_BUILTIN:
2381                                 if (bte->stub == NULL) {
2382                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2383                                 }
2384                                 else {
2385                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2386                                 }
2387                                 M_CALL(REG_ITMP1);
2388                                 break;
2389
2390                         case ICMD_INVOKESPECIAL:
2391                                 emit_nullpointer_check(cd, iptr, REG_A0);
2392                                 /* fall through */
2393
2394                         case ICMD_INVOKESTATIC:
2395                                 if (lm == NULL) {
2396                                         disp = dseg_add_unique_address(cd, um);
2397
2398                                         patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
2399                                                                                   um, disp);
2400                                 }
2401                                 else {
2402                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2403                                 }
2404
2405                                 M_ALD(REG_ITMP2, RIP, disp);
2406                                 M_CALL(REG_ITMP2);
2407                                 break;
2408
2409                         case ICMD_INVOKEVIRTUAL:
2410                                 if (lm == NULL) {
2411                                         patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
2412
2413                                         s1 = 0;
2414                                 }
2415                                 else {
2416                                         s1 = OFFSET(vftbl_t, table[0]) +
2417                                                 sizeof(methodptr) * lm->vftblindex;
2418                                 }
2419
2420                                 /* implicit null-pointer check */
2421                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2422                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2423                                 M_CALL(REG_ITMP3);
2424                                 break;
2425
2426                         case ICMD_INVOKEINTERFACE:
2427                                 if (lm == NULL) {
2428                                         patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
2429
2430                                         s1 = 0;
2431                                         s2 = 0;
2432                                 }
2433                                 else {
2434                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2435                                                 sizeof(methodptr) * lm->clazz->index;
2436
2437                                         s2 = sizeof(methodptr) * (lm - lm->clazz->methods);
2438                                 }
2439
2440                                 /* implicit null-pointer check */
2441                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2442                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2443                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2444                                 M_CALL(REG_ITMP3);
2445                                 break;
2446                         }
2447
2448                         /* generate method profiling code */
2449
2450                         PROFILE_CYCLE_START;
2451
2452                         /* store size of call code in replacement point */
2453
2454                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2455                         REPLACEMENT_POINT_FORGC_BUILTIN_RETURN(cd, iptr);
2456
2457                         /* store return value */
2458
2459                         switch (md->returntype.type) {
2460                         case TYPE_INT:
2461                         case TYPE_LNG:
2462                         case TYPE_ADR:
2463                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2464                                 M_INTMOVE(REG_RESULT, s1);
2465                                 emit_store_dst(jd, iptr, s1);
2466                                 break;
2467                         case TYPE_FLT:
2468                         case TYPE_DBL:
2469                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2470                                 M_FLTMOVE(REG_FRESULT, s1);
2471                                 emit_store_dst(jd, iptr, s1);
2472                                 break;
2473                         default:
2474                                 /* TYPE_VOID */
2475                                 break;
2476                         }
2477                         break;
2478
2479
2480                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2481
2482                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2483                                 /* object type cast-check */
2484
2485                                 classinfo *super;
2486                                 s4         superindex;
2487
2488                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2489                                         super      = NULL;
2490                                         superindex = 0;
2491                                 }
2492                                 else {
2493                                         super      = iptr->sx.s23.s3.c.cls;
2494                                         superindex = super->index;
2495                                 }
2496
2497                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2498                                         CODEGEN_CRITICAL_SECTION_NEW;
2499
2500                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2501
2502                                 /* if class is not resolved, check which code to call */
2503
2504                                 if (super == NULL) {
2505                                         M_TEST(s1);
2506                                         emit_label_beq(cd, BRANCH_LABEL_1);
2507
2508                                         patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2509                                                                                   iptr->sx.s23.s3.c.ref, 0);
2510
2511                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2512                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2513                                         emit_label_beq(cd, BRANCH_LABEL_2);
2514                                 }
2515
2516                                 /* interface checkcast code */
2517
2518                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2519                                         if (super != NULL) {
2520                                                 M_TEST(s1);
2521                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2522                                         }
2523
2524                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2525
2526                                         if (super == NULL) {
2527                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
2528                                                                                           iptr->sx.s23.s3.c.ref,
2529                                                                                           0);
2530                                         }
2531
2532                                         M_ILD32(REG_ITMP3,
2533                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2534                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2535                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2536
2537                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2538                                                         OFFSET(vftbl_t, interfacetable[0]) -
2539                                                         superindex * sizeof(methodptr*));
2540                                         M_TEST(REG_ITMP3);
2541                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2542
2543                                         if (super == NULL)
2544                                                 emit_label_br(cd, BRANCH_LABEL_4);
2545                                         else
2546                                                 emit_label(cd, BRANCH_LABEL_3);
2547                                 }
2548
2549                                 /* class checkcast code */
2550
2551                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2552                                         if (super == NULL) {
2553                                                 emit_label(cd, BRANCH_LABEL_2);
2554
2555                                                 cr   = iptr->sx.s23.s3.c.ref;
2556                                                 disp = dseg_add_unique_address(cd, cr);
2557
2558                                                 patcher_add_patch_ref(jd,
2559                                                                                           PATCHER_resolve_classref_to_vftbl,
2560                                                                                           cr, disp);
2561                                         }
2562                                         else {
2563                                                 M_TEST(s1);
2564                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2565
2566                                                 disp = dseg_add_address(cd, super->vftbl);
2567                                         }
2568
2569                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2570                                         M_ALD(REG_ITMP3, RIP, disp);
2571
2572                                         CODEGEN_CRITICAL_SECTION_START;
2573
2574                                         M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2575
2576                                         /*                                      if (s1 != REG_ITMP1) { */
2577                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2578                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2579                                         /*                                                                                              REG_ITMP1); */
2580                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2581                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2582                                         /*                                                                                              REG_ITMP3); */
2583                                         /*  #if defined(ENABLE_THREADS) */
2584                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2585                                         /*  #endif */
2586                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2587
2588                                         /*                                      } else { */
2589
2590                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
2591                                         M_ISUB(REG_ITMP3, REG_ITMP2);
2592                                         M_ALD(REG_ITMP3, RIP, disp);
2593                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2594                                         /*                                      } */
2595
2596                                         CODEGEN_CRITICAL_SECTION_END;
2597
2598                                         M_ICMP(REG_ITMP3, REG_ITMP2);
2599                                         emit_classcast_check(cd, iptr, BRANCH_UGT, REG_ITMP3, s1);
2600
2601                                         if (super != NULL)
2602                                                 emit_label(cd, BRANCH_LABEL_5);
2603                                 }
2604
2605                                 if (super == NULL) {
2606                                         emit_label(cd, BRANCH_LABEL_1);
2607                                         emit_label(cd, BRANCH_LABEL_4);
2608                                 }
2609
2610                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2611                         }
2612                         else {
2613                                 /* array type cast-check */
2614
2615                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2616                                 M_INTMOVE(s1, REG_A0);
2617
2618                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2619                                         cr   = iptr->sx.s23.s3.c.ref;
2620                                         disp = dseg_add_unique_address(cd, cr);
2621
2622                                         patcher_add_patch_ref(jd,
2623                                                                                   PATCHER_resolve_classref_to_classinfo,
2624                                                                                   cr, disp);
2625                                 }
2626                                 else {
2627                                         disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2628                                 }
2629
2630                                 M_ALD(REG_A1, RIP, disp);
2631                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2632                                 M_CALL(REG_ITMP1);
2633
2634                                 /* s1 may have been destroyed over the function call */
2635                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2636                                 M_TEST(REG_RESULT);
2637                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2638
2639                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2640                         }
2641
2642                         M_INTMOVE(s1, d);
2643                         emit_store_dst(jd, iptr, d);
2644                         break;
2645
2646                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2647
2648                         {
2649                         classinfo *super;
2650                         s4         superindex;
2651
2652                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2653                                 super      = NULL;
2654                                 superindex = 0;
2655                         }
2656                         else {
2657                                 super      = iptr->sx.s23.s3.c.cls;
2658                                 superindex = super->index;
2659                         }
2660
2661                         if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2662                                 CODEGEN_CRITICAL_SECTION_NEW;
2663
2664                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2665                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2666
2667                         if (s1 == d) {
2668                                 M_INTMOVE(s1, REG_ITMP1);
2669                                 s1 = REG_ITMP1;
2670                         }
2671
2672                         M_CLR(d);
2673
2674                         /* if class is not resolved, check which code to call */
2675
2676                         if (super == NULL) {
2677                                 M_TEST(s1);
2678                                 emit_label_beq(cd, BRANCH_LABEL_1);
2679
2680                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2681                                                                           iptr->sx.s23.s3.c.ref, 0);
2682
2683                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2684                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2685                                 emit_label_beq(cd, BRANCH_LABEL_2);
2686                         }
2687
2688                         /* interface instanceof code */
2689
2690                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2691                                 if (super != NULL) {
2692                                         M_TEST(s1);
2693                                         emit_label_beq(cd, BRANCH_LABEL_3);
2694                                 }
2695
2696                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2697
2698                                 if (super == NULL) {
2699                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
2700                                                                                   iptr->sx.s23.s3.c.ref, 0);
2701                                 }
2702
2703                                 M_ILD32(REG_ITMP3,
2704                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2705                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2706
2707                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2708
2709                                 M_BLE(a);
2710                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2711                                                 OFFSET(vftbl_t, interfacetable[0]) -
2712                                                 superindex * sizeof(methodptr*));
2713                                 M_TEST(REG_ITMP1);
2714                                 M_SETNE(d);
2715
2716                                 if (super == NULL)
2717                                         emit_label_br(cd, BRANCH_LABEL_4);
2718                                 else
2719                                         emit_label(cd, BRANCH_LABEL_3);
2720                         }
2721
2722                         /* class instanceof code */
2723
2724                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2725                                 if (super == NULL) {
2726                                         emit_label(cd, BRANCH_LABEL_2);
2727
2728                                         cr   = iptr->sx.s23.s3.c.ref;
2729                                         disp = dseg_add_unique_address(cd, cr);
2730
2731                                         patcher_add_patch_ref(jd,
2732                                                                                   PATCHER_resolve_classref_to_vftbl,
2733                                                                                   cr, disp);
2734                                 }
2735                                 else {
2736                                         M_TEST(s1);
2737                                         emit_label_beq(cd, BRANCH_LABEL_5);
2738
2739                                         disp = dseg_add_address(cd, super->vftbl);
2740                                 }
2741
2742                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2743                                 M_ALD(REG_ITMP2, RIP, disp);
2744
2745                                 CODEGEN_CRITICAL_SECTION_START;
2746
2747                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
2748                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2749                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2750
2751                                 CODEGEN_CRITICAL_SECTION_END;
2752
2753                                 M_ISUB(REG_ITMP2, REG_ITMP1);
2754                                 M_CLR(d); /* may be REG_ITMP2 */
2755                                 M_ICMP(REG_ITMP3, REG_ITMP1);
2756                                 M_SETULE(d);
2757
2758                                 if (super != NULL)
2759                                         emit_label(cd, BRANCH_LABEL_5);
2760                         }
2761
2762                         if (super == NULL) {
2763                                 emit_label(cd, BRANCH_LABEL_1);
2764                                 emit_label(cd, BRANCH_LABEL_4);
2765                         }
2766
2767                         emit_store_dst(jd, iptr, d);
2768                         }
2769                         break;
2770
2771                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2772
2773                         /* check for negative sizes and copy sizes to stack if necessary  */
2774
2775                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2776
2777                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2778
2779                                 /* copy SAVEDVAR sizes to stack */
2780                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2781
2782                                 /* Already Preallocated? */
2783                                 if (!(var->flags & PREALLOC)) {
2784                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2785                                         M_LST(s2, REG_SP, s1 * 8);
2786                                 }
2787                         }
2788
2789                         /* a0 = dimension count */
2790
2791                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2792
2793                         /* is a patcher function set? */
2794
2795                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2796                                 cr   = iptr->sx.s23.s3.c.ref;
2797                                 disp = dseg_add_unique_address(cd, cr);
2798
2799                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
2800                                                                           cr, disp);
2801                         }
2802                         else {
2803                                 disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2804                         }
2805
2806                         /* a1 = classinfo */
2807
2808                         M_ALD(REG_A1, RIP, disp);
2809
2810                         /* a2 = pointer to dimensions = stack pointer */
2811
2812                         M_MOV(REG_SP, REG_A2);
2813
2814                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2815                         M_CALL(REG_ITMP1);
2816
2817                         /* check for exception before result assignment */
2818
2819                         emit_exception_check(cd, iptr);
2820
2821                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2822                         M_INTMOVE(REG_RESULT, s1);
2823                         emit_store_dst(jd, iptr, s1);
2824                         break;
2825
2826                 default:
2827                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2828                                                                                    iptr->opc);
2829                         return false;
2830         } /* switch */
2831
2832         } /* for instruction */
2833
2834         MCODECHECK(512); /* XXX require a lower number? */
2835
2836         /* At the end of a basic block we may have to append some nops,
2837            because the patcher stub calling code might be longer than the
2838            actual instruction. So codepatching does not change the
2839            following block unintentionally. */
2840
2841         if (cd->mcodeptr < cd->lastmcodeptr) {
2842                 while (cd->mcodeptr < cd->lastmcodeptr) {
2843                         M_NOP;
2844                 }
2845         }
2846
2847         } /* if (bptr -> flags >= BBREACHED) */
2848         } /* for basic block */
2849
2850         /* Generate patcher traps. */
2851
2852         emit_patcher_traps(jd);
2853
2854         /* everything's ok */
2855
2856         return true;
2857 }
2858
2859
2860 /* codegen_emit_stub_native ****************************************************
2861
2862    Emits a stub routine which calls a native method.
2863
2864 *******************************************************************************/
2865
2866 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f, int skipparams)
2867 {
2868         methodinfo  *m;
2869         codeinfo    *code;
2870         codegendata *cd;
2871         methoddesc  *md;
2872         int          i, j;
2873         int          s1, s2;
2874         int          disp;
2875
2876         /* Sanity check. */
2877
2878         assert(f != NULL);
2879
2880         /* Get required compiler data. */
2881
2882         m    = jd->m;
2883         code = jd->code;
2884         cd   = jd->cd;
2885
2886         /* initialize variables */
2887
2888         md = m->parseddesc;
2889
2890         /* calculate stack frame size */
2891
2892         cd->stackframesize =
2893                 sizeof(stackframeinfo_t) / SIZEOF_VOID_P +
2894                 sizeof(localref_table) / SIZEOF_VOID_P +
2895                 md->paramcount +
2896                 (md->returntype.type == TYPE_VOID ? 0 : 1) +
2897                 nmd->memuse;
2898
2899         ALIGN_ODD(cd->stackframesize);              /* keep stack 16-byte aligned */
2900
2901         /* create method header */
2902
2903         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2904         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2905         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2906         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2907         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2908
2909 #if defined(ENABLE_PROFILING)
2910         /* generate native method profiling code */
2911
2912         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
2913                 /* count frequency */
2914
2915                 M_MOV_IMM(code, REG_ITMP3);
2916                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
2917         }
2918 #endif
2919
2920         /* generate stub code */
2921
2922         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2923
2924 #if defined(ENABLE_GC_CACAO)
2925         /* Save callee saved integer registers in stackframeinfo (GC may
2926            need to recover them during a collection). */
2927
2928         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
2929                 OFFSET(stackframeinfo_t, intregs);
2930
2931         for (i = 0; i < INT_SAV_CNT; i++)
2932                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2933 #endif
2934
2935         /* save integer and float argument registers */
2936
2937         for (i = 0; i < md->paramcount; i++) {
2938                 if (!md->params[i].inmemory) {
2939                         s1 = md->params[i].regoff;
2940
2941                         switch (md->paramtypes[i].type) {
2942                         case TYPE_INT:
2943                         case TYPE_LNG:
2944                         case TYPE_ADR:
2945                                 M_LST(s1, REG_SP, i * 8);
2946                                 break;
2947                         case TYPE_FLT:
2948                         case TYPE_DBL:
2949                                 M_DST(s1, REG_SP, i * 8);
2950                                 break;
2951                         }
2952                 }
2953         }
2954
2955         /* create dynamic stack info */
2956
2957         M_MOV(REG_SP, REG_A0);
2958         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
2959         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
2960         M_CALL(REG_ITMP1);
2961
2962         /* remember class argument */
2963
2964         if (m->flags & ACC_STATIC)
2965                 M_MOV(REG_RESULT, REG_ITMP2);
2966
2967         /* restore integer and float argument registers */
2968
2969         for (i = 0; i < md->paramcount; i++) {
2970                 if (!md->params[i].inmemory) {
2971                         s1 = md->params[i].regoff;
2972
2973                         switch (md->paramtypes[i].type) {
2974                         case TYPE_INT:
2975                         case TYPE_LNG:
2976                         case TYPE_ADR:
2977                                 M_LLD(s1, REG_SP, i * 8);
2978                                 break;
2979                         case TYPE_FLT:
2980                         case TYPE_DBL:
2981                                 M_DLD(s1, REG_SP, i * 8);
2982                                 break;
2983                         }
2984                 }
2985         }
2986
2987         /* Copy or spill arguments to new locations. */
2988
2989         for (i = md->paramcount - 1, j = i + skipparams; i >= 0; i--, j--) {
2990                 s2 = nmd->params[j].regoff;
2991
2992                 switch (md->paramtypes[i].type) {
2993                 case TYPE_INT:
2994                 case TYPE_LNG:
2995                 case TYPE_ADR:
2996                         if (!md->params[i].inmemory) {
2997                                 s1 = md->params[i].regoff;
2998
2999                                 if (!nmd->params[j].inmemory)
3000                                         M_INTMOVE(s1, s2);
3001                                 else
3002                                         M_LST(s1, REG_SP, s2);
3003                         }
3004                         else {
3005                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3006                                 M_LLD(REG_ITMP1, REG_SP, s1);
3007                                 M_LST(REG_ITMP1, REG_SP, s2);
3008                         }
3009                         break;
3010                 case TYPE_FLT:
3011                         /* We only copy spilled float arguments, as the float
3012                            argument registers keep unchanged. */
3013
3014                         if (md->params[i].inmemory) {
3015                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3016
3017                                 M_FLD(REG_FTMP1, REG_SP, s1);
3018                                 M_FST(REG_FTMP1, REG_SP, s2);
3019                         }
3020                         break;
3021                 case TYPE_DBL:
3022                         if (md->params[i].inmemory) {
3023                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3024                                 M_DLD(REG_FTMP1, REG_SP, s1);
3025                                 M_DST(REG_FTMP1, REG_SP, s2);
3026                         }
3027                         break;
3028                 }
3029         }
3030
3031         /* Handle native Java methods. */
3032
3033         if (m->flags & ACC_NATIVE) {
3034                 /* put class into second argument register */
3035
3036                 if (m->flags & ACC_STATIC)
3037                         M_MOV(REG_ITMP2, REG_A1);
3038
3039                 /* put env into first argument register */
3040
3041                 M_MOV_IMM(_Jv_env, REG_A0);
3042         }
3043
3044         /* Call the native function. */
3045
3046         disp = dseg_add_functionptr(cd, f);
3047         M_ALD(REG_ITMP1, RIP, disp);
3048         M_CALL(REG_ITMP1);
3049
3050         /* save return value */
3051
3052         switch (md->returntype.type) {
3053         case TYPE_INT:
3054         case TYPE_LNG:
3055         case TYPE_ADR:
3056                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3057                 break;
3058         case TYPE_FLT:
3059         case TYPE_DBL:
3060                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3061                 break;
3062         case TYPE_VOID:
3063                 break;
3064         }
3065
3066         /* remove native stackframe info */
3067
3068         M_MOV(REG_SP, REG_A0);
3069         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3070         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3071         M_CALL(REG_ITMP1);
3072         M_MOV(REG_RESULT, REG_ITMP3);
3073
3074         /* restore return value */
3075
3076         switch (md->returntype.type) {
3077         case TYPE_INT:
3078         case TYPE_LNG:
3079         case TYPE_ADR:
3080                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3081                 break;
3082         case TYPE_FLT:
3083         case TYPE_DBL:
3084                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3085                 break;
3086         case TYPE_VOID:
3087                 break;
3088         }
3089
3090 #if defined(ENABLE_GC_CACAO)
3091         /* Restore callee saved integer registers from stackframeinfo (GC
3092            might have modified them during a collection). */
3093
3094         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
3095                 OFFSET(stackframeinfo_t, intregs);
3096
3097         for (i = 0; i < INT_SAV_CNT; i++)
3098                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3099 #endif
3100
3101         /* remove stackframe */
3102
3103         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3104
3105         /* test for exception */
3106
3107         M_TEST(REG_ITMP3);
3108         M_BNE(1);
3109         M_RET;
3110
3111         /* handle exception */
3112
3113         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3114         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3115         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3116
3117         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3118         M_JMP(REG_ITMP3);
3119 }
3120
3121
3122 /*
3123  * These are local overrides for various environment variables in Emacs.
3124  * Please do not remove this and leave it at the end of the file, where
3125  * Emacs will automagically detect them.
3126  * ---------------------------------------------------------------------
3127  * Local variables:
3128  * mode: c
3129  * indent-tabs-mode: t
3130  * c-basic-offset: 4
3131  * tab-width: 4
3132  * End:
3133  * vim:noexpandtab:sw=4:ts=4:
3134  */