src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007, 2008
   4    CACAOVM - Verein zur Foerderung der freien virtuellen Maschine CACAO
   5
   6    This file is part of CACAO.
   7
   8    This program is free software; you can redistribute it and/or
   9    modify it under the terms of the GNU General Public License as
  10    published by the Free Software Foundation; either version 2, or (at
  11    your option) any later version.
  12
  13    This program is distributed in the hope that it will be useful, but
  14    WITHOUT ANY WARRANTY; without even the implied warranty of
  15    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16    General Public License for more details.
  17
  18    You should have received a copy of the GNU General Public License
  19    along with this program; if not, write to the Free Software
  20    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  21    02110-1301, USA.
  22
  23 */
  24
  25
  26 #include "config.h"
  27
  28 #include <assert.h>
  29 #include <stdio.h>
  30 #include <stdint.h>
  31
  32 #include "vm/types.h"
  33
  34 #include "md-abi.h"
  35
  36 #include "vm/jit/x86_64/arch.h"
  37 #include "vm/jit/x86_64/codegen.h"
  38 #include "vm/jit/x86_64/emit.h"
  39
  40 #include "mm/memory.h"
  41
  42 #include "native/jni.h"
  43 #include "native/localref.h"
  44 #include "native/native.h"
  45
  46 #include "threads/lock-common.h"
  47
  48 #include "vm/builtin.h"
  49 #include "vm/exceptions.hpp"
  50 #include "vm/global.h"
  51 #include "vm/primitive.hpp"
  52 #include "vm/string.hpp"
  53 #include "vm/vm.hpp"
  54
  55 #include "vm/jit/abi.h"
  56 #include "vm/jit/asmpart.h"
  57 #include "vm/jit/code.h"
  58 #include "vm/jit/codegen-common.h"
  59 #include "vm/jit/dseg.h"
  60 #include "vm/jit/emit-common.h"
  61 #include "vm/jit/jit.h"
  62 #include "vm/jit/linenumbertable.h"
  63 #include "vm/jit/methodheader.h"
  64 #include "vm/jit/parse.h"
  65 #include "vm/jit/patcher-common.h"
  66 #include "vm/jit/reg.h"
  67 #include "vm/jit/replace.h"
  68 #include "vm/jit/stacktrace.hpp"
  69 #include "vm/jit/trap.h"
  70
  71 #if defined(ENABLE_LSRA)
  72 # include "vm/jit/allocator/lsra.h"
  73 #endif
  74
  75 #include "vmcore/loader.h"
  76 #include "vmcore/options.h"
  77 #include "vmcore/statistics.h"
  78
  79
  80 /* codegen_emit ****************************************************************
  81
  82    Generates machine code.
  83
  84 *******************************************************************************/
  85
  86 bool codegen_emit(jitdata *jd)
  87 {
  88         methodinfo         *m;
  89         codeinfo           *code;
  90         codegendata        *cd;
  91         registerdata       *rd;
  92         s4                  len, s1, s2, s3, d, disp;
  93         u2                  currentline;
  94         ptrint              a;
  95         varinfo            *var, *dst;
  96         basicblock         *bptr;
  97         instruction        *iptr;
  98         constant_classref  *cr;
  99         unresolved_class   *uc;
 100         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 101         unresolved_method  *um;
 102         builtintable_entry *bte;
 103         methoddesc         *md;
 104         fieldinfo          *fi;
 105         unresolved_field   *uf;
 106         s4                  fieldtype;
 107         s4                 varindex;
 108
 109         /* get required compiler data */
 110
 111         m    = jd->m;
 112         code = jd->code;
 113         cd   = jd->cd;
 114         rd   = jd->rd;
 115
 116         /* prevent compiler warnings */
 117
 118         d   = 0;
 119         lm  = NULL;
 120         um  = NULL;
 121         bte = NULL;
 122
 123         {
 124         s4 i, p, t, l;
 125         s4 savedregs_num;
 126
 127         savedregs_num = 0;
 128
 129         /* space to save used callee saved registers */
 130
 131         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 132         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 133
 134         cd->stackframesize = rd->memuse + savedregs_num;
 135
 136 #if defined(ENABLE_THREADS)
 137         /* space to save argument of monitor_enter */
 138
 139         if (checksync && code_is_synchronized(code))
 140                 cd->stackframesize++;
 141 #endif
 142
 143         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 144            native code e.g. libc or jni (alignment problems with
 145            movaps). */
 146
 147         if (!code_is_leafmethod(code) || opt_verbosecall)
 148                 cd->stackframesize |= 0x1;
 149
 150         /* create method header */
 151
 152         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 153         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 154
 155         code->synchronizedoffset = rd->memuse * 8;
 156
 157         if (code_is_leafmethod(code))
 158                 (void) dseg_add_unique_s4(cd, 1);                  /* IsLeaf          */
 159         else
 160                 (void) dseg_add_unique_s4(cd, 0);                  /* IsLeaf          */
 161
 162         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 163         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 164
 165 #if defined(ENABLE_PROFILING)
 166         /* generate method profiling code */
 167
 168         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 169                 /* count frequency */
 170
 171                 M_MOV_IMM(code, REG_ITMP3);
 172                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 173
 174                 PROFILE_CYCLE_START;
 175         }
 176 #endif
 177
 178         /* create stack frame (if necessary) */
 179
 180         if (cd->stackframesize)
 181                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 182
 183         /* save used callee saved registers */
 184
 185         p = cd->stackframesize;
 186         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 187                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 188         }
 189         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 190                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 191         }
 192
 193         /* take arguments out of register or stack frame */
 194
 195         md = m->parseddesc;
 196
 197         for (p = 0, l = 0; p < md->paramcount; p++) {
 198                 t = md->paramtypes[p].type;
 199
 200                 varindex = jd->local_map[l * 5 + t];
 201
 202                 l++;
 203                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 204                         l++;
 205
 206                 if (varindex == UNUSED)
 207                         continue;
 208
 209                 var = VAR(varindex);
 210
 211                 s1 = md->params[p].regoff;
 212
 213                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 214                         if (!md->params[p].inmemory) {           /* register arguments    */
 215                                 if (!IS_INMEMORY(var->flags))
 216                                         M_INTMOVE(s1, var->vv.regoff);
 217                                 else
 218                                     M_LST(s1, REG_SP, var->vv.regoff);
 219                         }
 220                         else {                                 /* stack arguments       */
 221                                 if (!IS_INMEMORY(var->flags))
 222                                         /* + 8 for return address */
 223                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 224                                 else
 225                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 226                         }
 227                 }
 228                 else {                                     /* floating args         */
 229                         if (!md->params[p].inmemory) {           /* register arguments    */
 230                                 if (!IS_INMEMORY(var->flags))
 231                                         M_FLTMOVE(s1, var->vv.regoff);
 232                                 else
 233                                         M_DST(s1, REG_SP, var->vv.regoff);
 234                         }
 235                         else {                                 /* stack arguments       */
 236                                 if (!IS_INMEMORY(var->flags))
 237                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 238                                 else
 239                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 240                         }
 241                 }
 242         }
 243
 244         /* save monitorenter argument */
 245
 246 #if defined(ENABLE_THREADS)
 247         if (checksync && code_is_synchronized(code)) {
 248                 /* stack offset for monitor argument */
 249
 250                 s1 = rd->memuse;
 251
 252                 if (opt_verbosecall) {
 253                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 254
 255                         for (p = 0; p < INT_ARG_CNT; p++)
 256                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 257
 258                         for (p = 0; p < FLT_ARG_CNT; p++)
 259                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 260
 261                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 262                 }
 263
 264                 /* decide which monitor enter function to call */
 265
 266                 if (m->flags & ACC_STATIC) {
 267                         M_MOV_IMM(&m->clazz->object.header, REG_A0);
 268                 }
 269                 else {
 270                         M_TEST(REG_A0);
 271                         M_BNE(8);
 272                         M_ALD_MEM(REG_A0, TRAP_NullPointerException);
 273                 }
 274
 275                 M_AST(REG_A0, REG_SP, s1 * 8);
 276                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 277                 M_CALL(REG_ITMP1);
 278
 279                 if (opt_verbosecall) {
 280                         for (p = 0; p < INT_ARG_CNT; p++)
 281                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 282
 283                         for (p = 0; p < FLT_ARG_CNT; p++)
 284                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 285
 286                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 287                 }
 288         }
 289 #endif
 290
 291 #if !defined(NDEBUG)
 292         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 293                 emit_verbosecall_enter(jd);
 294 #endif /* !defined(NDEBUG) */
 295
 296         }
 297
 298         /* end of header generation */
 299
 300         /* create replacement points */
 301
 302         REPLACEMENT_POINTS_INIT(cd, jd);
 303
 304         /* walk through all basic blocks */
 305
 306         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 307
 308                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 309
 310                 if (bptr->flags >= BBREACHED) {
 311
 312                 /* branch resolving */
 313
 314                 codegen_resolve_branchrefs(cd, bptr);
 315
 316                 /* handle replacement points */
 317
 318                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 319
 320                 /* copy interface registers to their destination */
 321
 322                 len = bptr->indepth;
 323                 MCODECHECK(512);
 324
 325 #if defined(ENABLE_PROFILING)
 326                 /* generate basicblock profiling code */
 327
 328                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 329                         /* count frequency */
 330
 331                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 332                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 333
 334                         /* if this is an exception handler, start profiling again */
 335
 336                         if (bptr->type == BBTYPE_EXH)
 337                                 PROFILE_CYCLE_START;
 338                 }
 339 #endif
 340
 341 #if defined(ENABLE_LSRA)
 342                 if (opt_lsra) {
 343                         while (len) {
 344                                 len--;
 345                                 src = bptr->invars[len];
 346                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 347                                         if (bptr->type == BBTYPE_EXH) {
 348 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 349                                                 if (!IS_INMEMORY(src->flags))
 350                                                         d= src->vv.regoff;
 351                                                 else
 352                                                         d=REG_ITMP1;
 353                                                 M_INTMOVE(REG_ITMP1, d);
 354                                                 emit_store(jd, NULL, src, d);
 355                                         }
 356                                 }
 357                         }
 358
 359                 } else {
 360 #endif
 361
 362                 while (len) {
 363                         len--;
 364                         var = VAR(bptr->invars[len]);
 365                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 366                                 if (bptr->type == BBTYPE_EXH) {
 367                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 368                                         M_INTMOVE(REG_ITMP1, d);
 369                                         emit_store(jd, NULL, var, d);
 370                                 }
 371                         }
 372                         else {
 373                                 assert((var->flags & INOUT));
 374                         }
 375                 }
 376 #if defined(ENABLE_LSRA)
 377                 }
 378 #endif
 379                 /* walk through all instructions */
 380
 381                 len = bptr->icount;
 382                 currentline = 0;
 383
 384                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 385                         if (iptr->line != currentline) {
 386                                 linenumbertable_list_entry_add(cd, iptr->line);
 387                                 currentline = iptr->line;
 388                         }
 389
 390                         MCODECHECK(1024);                         /* 1KB should be enough */
 391
 392                 switch (iptr->opc) {
 393                 case ICMD_NOP:        /* ...  ==> ...                                 */
 394                 case ICMD_POP:        /* ..., value  ==> ...                          */
 395                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 396                         break;
 397
 398                 case ICMD_INLINE_START:
 399
 400                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 401                         break;
 402
 403                 case ICMD_INLINE_BODY:
 404
 405                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 406                         linenumbertable_list_entry_add_inline_start(cd, iptr);
 407                         linenumbertable_list_entry_add(cd, iptr->line);
 408                         break;
 409
 410                 case ICMD_INLINE_END:
 411
 412                         linenumbertable_list_entry_add_inline_end(cd, iptr);
 413                         linenumbertable_list_entry_add(cd, iptr->line);
 414                         break;
 415
 416                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 417
 418                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 419                         emit_nullpointer_check(cd, iptr, s1);
 420                         break;
 421
 422                 /* constant operations ************************************************/
 423
 424                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 425
 426                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 427                         ICONST(d, iptr->sx.val.i);
 428                         emit_store_dst(jd, iptr, d);
 429                         break;
 430
 431                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 432
 433                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 434                         LCONST(d, iptr->sx.val.l);
 435                         emit_store_dst(jd, iptr, d);
 436                         break;
 437
 438                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 439
 440                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 441                         disp = dseg_add_float(cd, iptr->sx.val.f);
 442                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 443                         emit_store_dst(jd, iptr, d);
 444                         break;
 445
 446                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 447
 448                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 449                         disp = dseg_add_double(cd, iptr->sx.val.d);
 450                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 451                         emit_store_dst(jd, iptr, d);
 452                         break;
 453
 454                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 455
 456                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 457
 458                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 459                                 cr   = iptr->sx.val.c.ref;
 460                                 disp = dseg_add_unique_address(cd, cr);
 461
 462 /*                              PROFILE_CYCLE_STOP; */
 463
 464                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
 465                                                                           cr, disp);
 466
 467 /*                              PROFILE_CYCLE_START; */
 468
 469                                 M_ALD(d, RIP, disp);
 470                         }
 471                         else {
 472                                 if (iptr->sx.val.anyptr == 0) {
 473                                         M_CLR(d);
 474                                 }
 475                                 else {
 476                                         disp = dseg_add_address(cd, iptr->sx.val.anyptr);
 477                                         M_ALD(d, RIP, disp);
 478                                 }
 479                         }
 480                         emit_store_dst(jd, iptr, d);
 481                         break;
 482
 483
 484                 /* load/store/copy/move operations ************************************/
 485
 486                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 487                 case ICMD_ALOAD:      /* s1 = local variable                          */
 488                 case ICMD_LLOAD:
 489                 case ICMD_FLOAD:
 490                 case ICMD_DLOAD:
 491                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 492                 case ICMD_LSTORE:
 493                 case ICMD_FSTORE:
 494                 case ICMD_DSTORE:
 495                 case ICMD_COPY:
 496                 case ICMD_MOVE:
 497
 498                         emit_copy(jd, iptr);
 499                         break;
 500
 501                 case ICMD_ASTORE:
 502                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 503                                 emit_copy(jd, iptr);
 504                         break;
 505
 506                 /* integer operations *************************************************/
 507
 508                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 509
 510                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 511                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 512                         M_INTMOVE(s1, d);
 513                         M_INEG(d);
 514                         emit_store_dst(jd, iptr, d);
 515                         break;
 516
 517                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 518
 519                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 520                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 521                         M_INTMOVE(s1, d);
 522                         M_LNEG(d);
 523                         emit_store_dst(jd, iptr, d);
 524                         break;
 525
 526                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 527
 528                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 529                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 530                         M_ISEXT(s1, d);
 531                         emit_store_dst(jd, iptr, d);
 532                         break;
 533
 534                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 535
 536                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 537                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 538                         M_IMOV(s1, d);
 539                         emit_store_dst(jd, iptr, d);
 540                         break;
 541
 542                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 543
 544                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 545                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 546                         M_BSEXT(s1, d);
 547                         emit_store_dst(jd, iptr, d);
 548                         break;
 549
 550                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 551
 552                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 553                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 554                         M_CZEXT(s1, d);
 555                         emit_store_dst(jd, iptr, d);
 556                         break;
 557
 558                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 559
 560                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 561                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 562                         M_SSEXT(s1, d);
 563                         emit_store_dst(jd, iptr, d);
 564                         break;
 565
 566
 567                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 568
 569                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 570                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 571                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 572                         if (s2 == d)
 573                                 M_IADD(s1, d);
 574                         else {
 575                                 M_INTMOVE(s1, d);
 576                                 M_IADD(s2, d);
 577                         }
 578                         emit_store_dst(jd, iptr, d);
 579                         break;
 580
 581                 case ICMD_IINC:
 582                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 583                                       /* sx.val.i = constant                             */
 584
 585                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 586                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 587
 588                         /* Using inc and dec is not faster than add (tested with
 589                            sieve). */
 590
 591                         M_INTMOVE(s1, d);
 592                         M_IADD_IMM(iptr->sx.val.i, d);
 593                         emit_store_dst(jd, iptr, d);
 594                         break;
 595
 596                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 597
 598                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 599                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 600                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 601                         if (s2 == d)
 602                                 M_LADD(s1, d);
 603                         else {
 604                                 M_INTMOVE(s1, d);
 605                                 M_LADD(s2, d);
 606                         }
 607                         emit_store_dst(jd, iptr, d);
 608                         break;
 609
 610                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 611                                       /* sx.val.l = constant                             */
 612
 613                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 614                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 615                         M_INTMOVE(s1, d);
 616                         if (IS_IMM32(iptr->sx.val.l))
 617                                 M_LADD_IMM(iptr->sx.val.l, d);
 618                         else {
 619                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 620                                 M_LADD(REG_ITMP2, d);
 621                         }
 622                         emit_store_dst(jd, iptr, d);
 623                         break;
 624
 625                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 626
 627                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 628                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 629                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 630                         if (s2 == d) {
 631                                 M_INTMOVE(s1, REG_ITMP1);
 632                                 M_ISUB(s2, REG_ITMP1);
 633                                 M_INTMOVE(REG_ITMP1, d);
 634                         } else {
 635                                 M_INTMOVE(s1, d);
 636                                 M_ISUB(s2, d);
 637                         }
 638                         emit_store_dst(jd, iptr, d);
 639                         break;
 640
 641                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 642                                       /* sx.val.i = constant                             */
 643
 644                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 645                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 646                         M_INTMOVE(s1, d);
 647                         M_ISUB_IMM(iptr->sx.val.i, d);
 648                         emit_store_dst(jd, iptr, d);
 649                         break;
 650
 651                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 652
 653                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 654                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 655                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 656                         if (s2 == d) {
 657                                 M_INTMOVE(s1, REG_ITMP1);
 658                                 M_LSUB(s2, REG_ITMP1);
 659                                 M_INTMOVE(REG_ITMP1, d);
 660                         } else {
 661                                 M_INTMOVE(s1, d);
 662                                 M_LSUB(s2, d);
 663                         }
 664                         emit_store_dst(jd, iptr, d);
 665                         break;
 666
 667                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 668                                       /* sx.val.l = constant                             */
 669
 670                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 671                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 672                         M_INTMOVE(s1, d);
 673                         if (IS_IMM32(iptr->sx.val.l))
 674                                 M_LSUB_IMM(iptr->sx.val.l, d);
 675                         else {
 676                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 677                                 M_LSUB(REG_ITMP2, d);
 678                         }
 679                         emit_store_dst(jd, iptr, d);
 680                         break;
 681
 682                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 683
 684                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 685                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 686                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 687                         if (s2 == d)
 688                                 M_IMUL(s1, d);
 689                         else {
 690                                 M_INTMOVE(s1, d);
 691                                 M_IMUL(s2, d);
 692                         }
 693                         emit_store_dst(jd, iptr, d);
 694                         break;
 695
 696                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 697                                       /* sx.val.i = constant                             */
 698
 699                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 700                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 701                         if (iptr->sx.val.i == 2) {
 702                                 M_INTMOVE(s1, d);
 703                                 M_ISLL_IMM(1, d);
 704                         } else
 705                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 706                         emit_store_dst(jd, iptr, d);
 707                         break;
 708
 709                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 710
 711                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 712                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 713                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 714                         if (s2 == d)
 715                                 M_LMUL(s1, d);
 716                         else {
 717                                 M_INTMOVE(s1, d);
 718                                 M_LMUL(s2, d);
 719                         }
 720                         emit_store_dst(jd, iptr, d);
 721                         break;
 722
 723                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 724                                       /* sx.val.l = constant                             */
 725
 726                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 727                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 728                         if (IS_IMM32(iptr->sx.val.l))
 729                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 730                         else {
 731                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 732                                 M_INTMOVE(s1, d);
 733                                 M_LMUL(REG_ITMP2, d);
 734                         }
 735                         emit_store_dst(jd, iptr, d);
 736                         break;
 737
 738                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 739
 740                         s1 = emit_load_s1(jd, iptr, RAX);
 741                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 742                         d = codegen_reg_of_dst(jd, iptr, RAX);
 743
 744                         M_INTMOVE(s1, RAX);
 745                         M_INTMOVE(s2, REG_ITMP3);
 746                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 747
 748                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 749
 750                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 751                         M_BNE(4 + 6);
 752                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 753                         M_BEQ(1 + 3);                                          /* 6 bytes */
 754
 755                         emit_cltd(cd);                                         /* 1 byte  */
 756                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 757
 758                         M_INTMOVE(RAX, d);
 759                         emit_store_dst(jd, iptr, d);
 760                         dst = VAROP(iptr->dst);
 761                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 762                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 763                         break;
 764
 765                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 766
 767                         s1 = emit_load_s1(jd, iptr, RAX);
 768                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 769                         d = codegen_reg_of_dst(jd, iptr, RDX);
 770
 771                         M_INTMOVE(s1, RAX);
 772                         M_INTMOVE(s2, REG_ITMP3);
 773                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 774
 775                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 776
 777                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 778                         M_BNE(3 + 4 + 6);
 779                         M_CLR(RDX);                                            /* 3 bytes */
 780                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 781                         M_BEQ(1 + 3);                                          /* 6 bytes */
 782
 783                         emit_cltd(cd);                                         /* 1 byte  */
 784                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 785
 786                         M_INTMOVE(RDX, d);
 787                         emit_store_dst(jd, iptr, d);
 788                         dst = VAROP(iptr->dst);
 789                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 790                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 791                         break;
 792
 793                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 794                                       /* sx.val.i = constant                             */
 795
 796                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 797                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 798                         M_INTMOVE(s1, REG_ITMP1);
 799                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 800                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 801                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 802                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 803                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 804                         emit_store_dst(jd, iptr, d);
 805                         break;
 806
 807                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 808                                       /* sx.val.i = constant                             */
 809
 810                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 811                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 812                         M_INTMOVE(s1, REG_ITMP1);
 813                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 814                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 815                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 816                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 817                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 818                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 819                         emit_store_dst(jd, iptr, d);
 820                         break;
 821
 822
 823                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 824
 825                         s1 = emit_load_s1(jd, iptr, RAX);
 826                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 827                         d = codegen_reg_of_dst(jd, iptr, RAX);
 828
 829                         M_INTMOVE(s1, RAX);
 830                         M_INTMOVE(s2, REG_ITMP3);
 831                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 832
 833                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 834
 835                         /* check as described in jvm spec */
 836                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 837                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 838                         M_BNE(4 + 6);
 839                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 840                         M_BEQ(2 + 3);                                          /* 6 bytes */
 841
 842                         emit_cqto(cd);                                         /* 2 bytes */
 843                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 844
 845                         M_INTMOVE(RAX, d);
 846                         emit_store_dst(jd, iptr, d);
 847                         dst = VAROP(iptr->dst);
 848                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 849                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 850                         break;
 851
 852                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 853
 854                         s1 = emit_load_s1(jd, iptr, RAX);
 855                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 856                         d = codegen_reg_of_dst(jd, iptr, RDX);
 857
 858                         M_INTMOVE(s1, RAX);
 859                         M_INTMOVE(s2, REG_ITMP3);
 860                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 861
 862                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 863
 864                         /* check as described in jvm spec */
 865                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 866                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 867                         M_BNE(3 + 4 + 6);
 868                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 869                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 870                         M_BEQ(2 + 3);                                          /* 6 bytes */
 871
 872                         emit_cqto(cd);                                         /* 2 bytes */
 873                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 874
 875                         M_INTMOVE(RDX, d);
 876                         emit_store_dst(jd, iptr, d);
 877                         dst = VAROP(iptr->dst);
 878                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 879                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 880                         break;
 881
 882                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 883                                       /* sx.val.i = constant                             */
 884
 885                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 886                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 887                         M_INTMOVE(s1, REG_ITMP1);
 888                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 889                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 890                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 891                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 892                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 893                         emit_store_dst(jd, iptr, d);
 894                         break;
 895
 896                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 897                                       /* sx.val.l = constant                             */
 898
 899                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 900                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 901                         M_INTMOVE(s1, REG_ITMP1);
 902                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 903                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 904                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 905                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 906                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 907                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 908                         emit_store_dst(jd, iptr, d);
 909                         break;
 910
 911                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 912
 913                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 914                         emit_ishift(jd, SHIFT_SHL, iptr);
 915                         break;
 916
 917                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 918                                       /* sx.val.i = constant                             */
 919
 920                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 921                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 922                         M_INTMOVE(s1, d);
 923                         M_ISLL_IMM(iptr->sx.val.i, d);
 924                         emit_store_dst(jd, iptr, d);
 925                         break;
 926
 927                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 928
 929                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 930                         emit_ishift(jd, SHIFT_SAR, iptr);
 931                         break;
 932
 933                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 934                                       /* sx.val.i = constant                             */
 935
 936                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 937                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 938                         M_INTMOVE(s1, d);
 939                         M_ISRA_IMM(iptr->sx.val.i, d);
 940                         emit_store_dst(jd, iptr, d);
 941                         break;
 942
 943                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 944
 945                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 946                         emit_ishift(jd, SHIFT_SHR, iptr);
 947                         break;
 948
 949                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 950                                       /* sx.val.i = constant                             */
 951
 952                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 953                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 954                         M_INTMOVE(s1, d);
 955                         M_ISRL_IMM(iptr->sx.val.i, d);
 956                         emit_store_dst(jd, iptr, d);
 957                         break;
 958
 959                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 960
 961                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 962                         emit_lshift(jd, SHIFT_SHL, iptr);
 963                         break;
 964
 965         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 966                                           /* sx.val.i = constant                             */
 967
 968                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 969                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 970                         M_INTMOVE(s1, d);
 971                         M_LSLL_IMM(iptr->sx.val.i, d);
 972                         emit_store_dst(jd, iptr, d);
 973                         break;
 974
 975                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 976
 977                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 978                         emit_lshift(jd, SHIFT_SAR, iptr);
 979                         break;
 980
 981                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 982                                       /* sx.val.i = constant                             */
 983
 984                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 985                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 986                         M_INTMOVE(s1, d);
 987                         M_LSRA_IMM(iptr->sx.val.i, d);
 988                         emit_store_dst(jd, iptr, d);
 989                         break;
 990
 991                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 992
 993                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 994                         emit_lshift(jd, SHIFT_SHR, iptr);
 995                         break;
 996
 997                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 998                                       /* sx.val.l = constant                             */
 999
1000                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1001                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1002                         M_INTMOVE(s1, d);
1003                         M_LSRL_IMM(iptr->sx.val.i, d);
1004                         emit_store_dst(jd, iptr, d);
1005                         break;
1006
1007                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1008
1009                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1010                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1011                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1012                         if (s2 == d)
1013                                 M_IAND(s1, d);
1014                         else {
1015                                 M_INTMOVE(s1, d);
1016                                 M_IAND(s2, d);
1017                         }
1018                         emit_store_dst(jd, iptr, d);
1019                         break;
1020
1021                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1022                                       /* sx.val.i = constant                             */
1023
1024                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1025                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1026                         M_INTMOVE(s1, d);
1027                         M_IAND_IMM(iptr->sx.val.i, d);
1028                         emit_store_dst(jd, iptr, d);
1029                         break;
1030
1031                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1032
1033                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1034                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1035                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1036                         if (s2 == d)
1037                                 M_LAND(s1, d);
1038                         else {
1039                                 M_INTMOVE(s1, d);
1040                                 M_LAND(s2, d);
1041                         }
1042                         emit_store_dst(jd, iptr, d);
1043                         break;
1044
1045                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1046                                       /* sx.val.l = constant                             */
1047
1048                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1049                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1050                         M_INTMOVE(s1, d);
1051                         if (IS_IMM32(iptr->sx.val.l))
1052                                 M_LAND_IMM(iptr->sx.val.l, d);
1053                         else {
1054                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1055                                 M_LAND(REG_ITMP2, d);
1056                         }
1057                         emit_store_dst(jd, iptr, d);
1058                         break;
1059
1060                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1061
1062                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1063                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1064                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1065                         if (s2 == d)
1066                                 M_IOR(s1, d);
1067                         else {
1068                                 M_INTMOVE(s1, d);
1069                                 M_IOR(s2, d);
1070                         }
1071                         emit_store_dst(jd, iptr, d);
1072                         break;
1073
1074                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1075                                       /* sx.val.i = constant                             */
1076
1077                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1078                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1079                         M_INTMOVE(s1, d);
1080                         M_IOR_IMM(iptr->sx.val.i, d);
1081                         emit_store_dst(jd, iptr, d);
1082                         break;
1083
1084                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1085
1086                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1087                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1088                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1089                         if (s2 == d)
1090                                 M_LOR(s1, d);
1091                         else {
1092                                 M_INTMOVE(s1, d);
1093                                 M_LOR(s2, d);
1094                         }
1095                         emit_store_dst(jd, iptr, d);
1096                         break;
1097
1098                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1099                                       /* sx.val.l = constant                             */
1100
1101                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1102                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1103                         M_INTMOVE(s1, d);
1104                         if (IS_IMM32(iptr->sx.val.l))
1105                                 M_LOR_IMM(iptr->sx.val.l, d);
1106                         else {
1107                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1108                                 M_LOR(REG_ITMP2, d);
1109                         }
1110                         emit_store_dst(jd, iptr, d);
1111                         break;
1112
1113                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1114
1115                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1116                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1117                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1118                         if (s2 == d)
1119                                 M_IXOR(s1, d);
1120                         else {
1121                                 M_INTMOVE(s1, d);
1122                                 M_IXOR(s2, d);
1123                         }
1124                         emit_store_dst(jd, iptr, d);
1125                         break;
1126
1127                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1128                                       /* sx.val.i = constant                             */
1129
1130                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1131                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1132                         M_INTMOVE(s1, d);
1133                         M_IXOR_IMM(iptr->sx.val.i, d);
1134                         emit_store_dst(jd, iptr, d);
1135                         break;
1136
1137                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1138
1139                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1140                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1141                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1142                         if (s2 == d)
1143                                 M_LXOR(s1, d);
1144                         else {
1145                                 M_INTMOVE(s1, d);
1146                                 M_LXOR(s2, d);
1147                         }
1148                         emit_store_dst(jd, iptr, d);
1149                         break;
1150
1151                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1152                                       /* sx.val.l = constant                             */
1153
1154                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1155                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1156                         M_INTMOVE(s1, d);
1157                         if (IS_IMM32(iptr->sx.val.l))
1158                                 M_LXOR_IMM(iptr->sx.val.l, d);
1159                         else {
1160                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1161                                 M_LXOR(REG_ITMP2, d);
1162                         }
1163                         emit_store_dst(jd, iptr, d);
1164                         break;
1165
1166
1167                 /* floating operations ************************************************/
1168
1169                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1170
1171                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1172                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1173                         disp = dseg_add_s4(cd, 0x80000000);
1174                         M_FLTMOVE(s1, d);
1175                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1176                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1177                         emit_store_dst(jd, iptr, d);
1178                         break;
1179
1180                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1181
1182                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1183                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1184                         disp = dseg_add_s8(cd, 0x8000000000000000);
1185                         M_FLTMOVE(s1, d);
1186                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1187                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1188                         emit_store_dst(jd, iptr, d);
1189                         break;
1190
1191                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1192
1193                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1194                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1195                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1196                         if (s2 == d)
1197                                 M_FADD(s1, d);
1198                         else {
1199                                 M_FLTMOVE(s1, d);
1200                                 M_FADD(s2, d);
1201                         }
1202                         emit_store_dst(jd, iptr, d);
1203                         break;
1204
1205                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1206
1207                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1208                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1209                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1210                         if (s2 == d)
1211                                 M_DADD(s1, d);
1212                         else {
1213                                 M_FLTMOVE(s1, d);
1214                                 M_DADD(s2, d);
1215                         }
1216                         emit_store_dst(jd, iptr, d);
1217                         break;
1218
1219                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1220
1221                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1222                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1223                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1224                         if (s2 == d) {
1225                                 M_FLTMOVE(s2, REG_FTMP2);
1226                                 s2 = REG_FTMP2;
1227                         }
1228                         M_FLTMOVE(s1, d);
1229                         M_FSUB(s2, d);
1230                         emit_store_dst(jd, iptr, d);
1231                         break;
1232
1233                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1234
1235                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1236                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1237                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1238                         if (s2 == d) {
1239                                 M_FLTMOVE(s2, REG_FTMP2);
1240                                 s2 = REG_FTMP2;
1241                         }
1242                         M_FLTMOVE(s1, d);
1243                         M_DSUB(s2, d);
1244                         emit_store_dst(jd, iptr, d);
1245                         break;
1246
1247                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1248
1249                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1250                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1251                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1252                         if (s2 == d)
1253                                 M_FMUL(s1, d);
1254                         else {
1255                                 M_FLTMOVE(s1, d);
1256                                 M_FMUL(s2, d);
1257                         }
1258                         emit_store_dst(jd, iptr, d);
1259                         break;
1260
1261                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1262
1263                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1264                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1265                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1266                         if (s2 == d)
1267                                 M_DMUL(s1, d);
1268                         else {
1269                                 M_FLTMOVE(s1, d);
1270                                 M_DMUL(s2, d);
1271                         }
1272                         emit_store_dst(jd, iptr, d);
1273                         break;
1274
1275                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1276
1277                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1278                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1279                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1280                         if (s2 == d) {
1281                                 M_FLTMOVE(s2, REG_FTMP2);
1282                                 s2 = REG_FTMP2;
1283                         }
1284                         M_FLTMOVE(s1, d);
1285                         M_FDIV(s2, d);
1286                         emit_store_dst(jd, iptr, d);
1287                         break;
1288
1289                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1290
1291                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1292                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1293                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1294                         if (s2 == d) {
1295                                 M_FLTMOVE(s2, REG_FTMP2);
1296                                 s2 = REG_FTMP2;
1297                         }
1298                         M_FLTMOVE(s1, d);
1299                         M_DDIV(s2, d);
1300                         emit_store_dst(jd, iptr, d);
1301                         break;
1302
1303                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1304
1305                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1306                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1307                         M_CVTIF(s1, d);
1308                         emit_store_dst(jd, iptr, d);
1309                         break;
1310
1311                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1312
1313                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1314                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1315                         M_CVTID(s1, d);
1316                         emit_store_dst(jd, iptr, d);
1317                         break;
1318
1319                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1320
1321                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1322                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1323                         M_CVTLF(s1, d);
1324                         emit_store_dst(jd, iptr, d);
1325                         break;
1326
1327                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1328
1329                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1330                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1331                         M_CVTLD(s1, d);
1332                         emit_store_dst(jd, iptr, d);
1333                         break;
1334
1335                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1336
1337                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1338                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1339                         M_CVTFI(s1, d);
1340                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1341                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1342                                 ((REG_RESULT == d) ? 0 : 3);
1343                         M_BNE(disp);
1344                         M_FLTMOVE(s1, REG_FTMP1);
1345                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1346                         M_CALL(REG_ITMP2);
1347                         M_INTMOVE(REG_RESULT, d);
1348                         emit_store_dst(jd, iptr, d);
1349                         break;
1350
1351                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1352
1353                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1354                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1355                         M_CVTDI(s1, d);
1356                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1357                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1358                                 ((REG_RESULT == d) ? 0 : 3);
1359                         M_BNE(disp);
1360                         M_FLTMOVE(s1, REG_FTMP1);
1361                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1362                         M_CALL(REG_ITMP2);
1363                         M_INTMOVE(REG_RESULT, d);
1364                         emit_store_dst(jd, iptr, d);
1365                         break;
1366
1367                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1368
1369                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1370                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1371                         M_CVTFL(s1, d);
1372                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1373                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1374                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1375                                 ((REG_RESULT == d) ? 0 : 3);
1376                         M_BNE(disp);
1377                         M_FLTMOVE(s1, REG_FTMP1);
1378                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1379                         M_CALL(REG_ITMP2);
1380                         M_INTMOVE(REG_RESULT, d);
1381                         emit_store_dst(jd, iptr, d);
1382                         break;
1383
1384                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1385
1386                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1387                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1388                         M_CVTDL(s1, d);
1389                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1390                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1391                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1392                                 ((REG_RESULT == d) ? 0 : 3);
1393                         M_BNE(disp);
1394                         M_FLTMOVE(s1, REG_FTMP1);
1395                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1396                         M_CALL(REG_ITMP2);
1397                         M_INTMOVE(REG_RESULT, d);
1398                         emit_store_dst(jd, iptr, d);
1399                         break;
1400
1401                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1402
1403                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1404                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1405                         M_CVTFD(s1, d);
1406                         emit_store_dst(jd, iptr, d);
1407                         break;
1408
1409                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1410
1411                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1412                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1413                         M_CVTDF(s1, d);
1414                         emit_store_dst(jd, iptr, d);
1415                         break;
1416
1417                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1418                                           /* == => 0, < => 1, > => -1 */
1419
1420                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1421                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1422                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1423                         M_CLR(d);
1424                         M_MOV_IMM(1, REG_ITMP1);
1425                         M_MOV_IMM(-1, REG_ITMP2);
1426                         emit_ucomiss_reg_reg(cd, s1, s2);
1427                         M_CMOVULT(REG_ITMP1, d);
1428                         M_CMOVUGT(REG_ITMP2, d);
1429                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1430                         emit_store_dst(jd, iptr, d);
1431                         break;
1432
1433                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1434                                           /* == => 0, < => 1, > => -1 */
1435
1436                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1437                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1438                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1439                         M_CLR(d);
1440                         M_MOV_IMM(1, REG_ITMP1);
1441                         M_MOV_IMM(-1, REG_ITMP2);
1442                         emit_ucomiss_reg_reg(cd, s1, s2);
1443                         M_CMOVULT(REG_ITMP1, d);
1444                         M_CMOVUGT(REG_ITMP2, d);
1445                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1446                         emit_store_dst(jd, iptr, d);
1447                         break;
1448
1449                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1450                                           /* == => 0, < => 1, > => -1 */
1451
1452                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1453                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1454                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1455                         M_CLR(d);
1456                         M_MOV_IMM(1, REG_ITMP1);
1457                         M_MOV_IMM(-1, REG_ITMP2);
1458                         emit_ucomisd_reg_reg(cd, s1, s2);
1459                         M_CMOVULT(REG_ITMP1, d);
1460                         M_CMOVUGT(REG_ITMP2, d);
1461                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1462                         emit_store_dst(jd, iptr, d);
1463                         break;
1464
1465                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1466                                           /* == => 0, < => 1, > => -1 */
1467
1468                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1469                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1470                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1471                         M_CLR(d);
1472                         M_MOV_IMM(1, REG_ITMP1);
1473                         M_MOV_IMM(-1, REG_ITMP2);
1474                         emit_ucomisd_reg_reg(cd, s1, s2);
1475                         M_CMOVULT(REG_ITMP1, d);
1476                         M_CMOVUGT(REG_ITMP2, d);
1477                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1478                         emit_store_dst(jd, iptr, d);
1479                         break;
1480
1481
1482                 /* memory operations **************************************************/
1483
1484                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1485
1486                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1487                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1488                         /* implicit null-pointer check */
1489                         M_ILD(d, s1, OFFSET(java_array_t, size));
1490                         emit_store_dst(jd, iptr, d);
1491                         break;
1492
1493                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1494
1495                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1496                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1497                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1498                         /* implicit null-pointer check */
1499                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1500                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1501                         emit_store_dst(jd, iptr, d);
1502                         break;
1503
1504                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1505
1506                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1507                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1508                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1509                         /* implicit null-pointer check */
1510                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1511                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1512                         emit_store_dst(jd, iptr, d);
1513                         break;
1514
1515                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1516
1517                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1518                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1519                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1520                         /* implicit null-pointer check */
1521                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1522                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1523                         emit_store_dst(jd, iptr, d);
1524                         break;
1525
1526                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1527
1528                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1529                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1530                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1531                         /* implicit null-pointer check */
1532                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1533                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1534                         emit_store_dst(jd, iptr, d);
1535                         break;
1536
1537                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1538
1539                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1540                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1541                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1542                         /* implicit null-pointer check */
1543                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1544                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1545                         emit_store_dst(jd, iptr, d);
1546                         break;
1547
1548                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1549
1550                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1551                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1552                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1553                         /* implicit null-pointer check */
1554                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1555                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1556                         emit_store_dst(jd, iptr, d);
1557                         break;
1558
1559                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1560
1561                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1562                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1563                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1564                         /* implicit null-pointer check */
1565                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1566                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1567                         emit_store_dst(jd, iptr, d);
1568                         break;
1569
1570                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1571
1572                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1573                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1574                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1575                         /* implicit null-pointer check */
1576                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1577                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1578                         emit_store_dst(jd, iptr, d);
1579                         break;
1580
1581
1582                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1583
1584                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1585                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1586                         /* implicit null-pointer check */
1587                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1588                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1589                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1590                         break;
1591
1592                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1593
1594                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1595                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1596                         /* implicit null-pointer check */
1597                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1598                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1599                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1600                         break;
1601
1602                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1603
1604                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1605                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1606                         /* implicit null-pointer check */
1607                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1608                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1609                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1610                         break;
1611
1612                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1613
1614                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1615                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1616                         /* implicit null-pointer check */
1617                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1618                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1619                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1620                         break;
1621
1622                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1623
1624                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1625                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1626                         /* implicit null-pointer check */
1627                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1628                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1629                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1630                         break;
1631
1632                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1633
1634                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1635                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1636                         /* implicit null-pointer check */
1637                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1638                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1639                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1640                         break;
1641
1642                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1643
1644                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1645                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1646                         /* implicit null-pointer check */
1647                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1648                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1649                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1650                         break;
1651
1652                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1653
1654                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1655                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1656                         /* implicit null-pointer check */
1657                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1658                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1659
1660                         M_MOV(s1, REG_A0);
1661                         M_MOV(s3, REG_A1);
1662                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
1663                         M_CALL(REG_ITMP1);
1664                         emit_arraystore_check(cd, iptr);
1665
1666                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1667                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1668                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1669                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1670                         break;
1671
1672
1673                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1674
1675                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1676                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1677                         /* implicit null-pointer check */
1678                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1679                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1680                         break;
1681
1682                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1683
1684                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1685                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1686                         /* implicit null-pointer check */
1687                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1688                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1689                         break;
1690
1691                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1692
1693                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1694                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1695                         /* implicit null-pointer check */
1696                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1697                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1698                         break;
1699
1700                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1701
1702                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1703                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1704                         /* implicit null-pointer check */
1705                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1706                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1707                         break;
1708
1709                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1710
1711                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1712                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1713                         /* implicit null-pointer check */
1714                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1715
1716                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1717                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1718                         }
1719                         else {
1720                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1721                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1722                         }
1723                         break;
1724
1725                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1726
1727                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1728                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1729                         /* implicit null-pointer check */
1730                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1731                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1732                         break;
1733
1734
1735                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1736
1737                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1738                                 uf        = iptr->sx.s23.s3.uf;
1739                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1740                                 disp      = dseg_add_unique_address(cd, uf);
1741
1742 /*                              PROFILE_CYCLE_STOP; */
1743
1744                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1745
1746 /*                              PROFILE_CYCLE_START; */
1747                         }
1748                         else {
1749                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1750                                 fieldtype = fi->type;
1751                                 disp      = dseg_add_address(cd, fi->value);
1752
1753                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1754                                         PROFILE_CYCLE_STOP;
1755
1756                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1757                                                                                   fi->clazz, 0);
1758
1759                                         PROFILE_CYCLE_START;
1760                                 }
1761                         }
1762
1763                         /* This approach is much faster than moving the field
1764                            address inline into a register. */
1765
1766                         M_ALD(REG_ITMP1, RIP, disp);
1767
1768                         switch (fieldtype) {
1769                         case TYPE_INT:
1770                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1771                                 M_ILD(d, REG_ITMP1, 0);
1772                                 break;
1773                         case TYPE_LNG:
1774                         case TYPE_ADR:
1775                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1776                                 M_LLD(d, REG_ITMP1, 0);
1777                                 break;
1778                         case TYPE_FLT:
1779                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1780                                 M_FLD(d, REG_ITMP1, 0);
1781                                 break;
1782                         case TYPE_DBL:
1783                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1784                                 M_DLD(d, REG_ITMP1, 0);
1785                                 break;
1786                         }
1787                         emit_store_dst(jd, iptr, d);
1788                         break;
1789
1790                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1791
1792                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1793                                 uf        = iptr->sx.s23.s3.uf;
1794                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1795                                 disp      = dseg_add_unique_address(cd, uf);
1796
1797 /*                              PROFILE_CYCLE_STOP; */
1798
1799                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1800
1801 /*                              PROFILE_CYCLE_START; */
1802                         }
1803                         else {
1804                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1805                                 fieldtype = fi->type;
1806                                 disp      = dseg_add_address(cd, fi->value);
1807
1808                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1809                                         PROFILE_CYCLE_STOP;
1810
1811                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1812                                                                                   fi->clazz, 0);
1813
1814                                         PROFILE_CYCLE_START;
1815                                 }
1816                         }
1817
1818                         /* This approach is much faster than moving the field
1819                            address inline into a register. */
1820
1821                         M_ALD(REG_ITMP1, RIP, disp);
1822
1823                         switch (fieldtype) {
1824                         case TYPE_INT:
1825                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1826                                 M_IST(s1, REG_ITMP1, 0);
1827                                 break;
1828                         case TYPE_LNG:
1829                         case TYPE_ADR:
1830                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1831                                 M_LST(s1, REG_ITMP1, 0);
1832                                 break;
1833                         case TYPE_FLT:
1834                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1835                                 M_FST(s1, REG_ITMP1, 0);
1836                                 break;
1837                         case TYPE_DBL:
1838                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1839                                 M_DST(s1, REG_ITMP1, 0);
1840                                 break;
1841                         }
1842                         break;
1843
1844                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1845                                           /* val = value (in current instruction)     */
1846                                           /* following NOP)                           */
1847
1848                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1849                                 uf        = iptr->sx.s23.s3.uf;
1850                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1851                                 disp      = dseg_add_unique_address(cd, uf);
1852
1853 /*                              PROFILE_CYCLE_STOP; */
1854
1855                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1856
1857 /*                              PROFILE_CYCLE_START; */
1858                         }
1859                         else {
1860                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1861                                 fieldtype = fi->type;
1862                                 disp      = dseg_add_address(cd, fi->value);
1863
1864                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1865                                         PROFILE_CYCLE_STOP;
1866
1867                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1868                                                                                   fi->clazz, 0);
1869
1870                                         PROFILE_CYCLE_START;
1871                                 }
1872                         }
1873
1874                         /* This approach is much faster than moving the field
1875                            address inline into a register. */
1876
1877                         M_ALD(REG_ITMP1, RIP, disp);
1878
1879                         switch (fieldtype) {
1880                         case TYPE_INT:
1881                         case TYPE_FLT:
1882                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1883                                 break;
1884                         case TYPE_LNG:
1885                         case TYPE_ADR:
1886                         case TYPE_DBL:
1887                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1888                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1889                                 else {
1890                                         M_MOV_IMM(iptr->sx.s23.s2.constval, REG_ITMP2);
1891                                         M_LST(REG_ITMP2, REG_ITMP1, 0);
1892                                 }
1893                                 break;
1894                         }
1895                         break;
1896
1897                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1898
1899                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1900
1901                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1902                                 uf        = iptr->sx.s23.s3.uf;
1903                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1904                                 disp      = 0;
1905
1906 /*                              PROFILE_CYCLE_STOP; */
1907
1908                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1909
1910 /*                              PROFILE_CYCLE_START; */
1911                         }
1912                         else {
1913                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1914                                 fieldtype = fi->type;
1915                                 disp      = fi->offset;
1916                         }
1917
1918                         /* implicit null-pointer check */
1919                         switch (fieldtype) {
1920                         case TYPE_INT:
1921                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1922                                 M_ILD32(d, s1, disp);
1923                                 break;
1924                         case TYPE_LNG:
1925                         case TYPE_ADR:
1926                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1927                                 M_LLD32(d, s1, disp);
1928                                 break;
1929                         case TYPE_FLT:
1930                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1931                                 M_FLD32(d, s1, disp);
1932                                 break;
1933                         case TYPE_DBL:
1934                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1935                                 M_DLD32(d, s1, disp);
1936                                 break;
1937                         }
1938                         emit_store_dst(jd, iptr, d);
1939                         break;
1940
1941                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1942
1943                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1944                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1945
1946                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1947                                 uf        = iptr->sx.s23.s3.uf;
1948                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1949                                 disp      = 0;
1950
1951 /*                              PROFILE_CYCLE_STOP; */
1952
1953                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1954
1955 /*                              PROFILE_CYCLE_START; */
1956                         }
1957                         else {
1958                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1959                                 fieldtype = fi->type;
1960                                 disp      = fi->offset;
1961                         }
1962
1963                         /* implicit null-pointer check */
1964                         switch (fieldtype) {
1965                         case TYPE_INT:
1966                                 M_IST32(s2, s1, disp);
1967                                 break;
1968                         case TYPE_LNG:
1969                         case TYPE_ADR:
1970                                 M_LST32(s2, s1, disp);
1971                                 break;
1972                         case TYPE_FLT:
1973                                 M_FST32(s2, s1, disp);
1974                                 break;
1975                         case TYPE_DBL:
1976                                 M_DST32(s2, s1, disp);
1977                                 break;
1978                         }
1979                         break;
1980
1981                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
1982                                           /* val = value (in current instruction)     */
1983                                           /* following NOP)                           */
1984
1985                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1986
1987                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1988                                 uf        = iptr->sx.s23.s3.uf;
1989                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1990                                 disp      = 0;
1991
1992 /*                              PROFILE_CYCLE_STOP; */
1993
1994                                 patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
1995
1996 /*                              PROFILE_CYCLE_START; */
1997                         }
1998                         else {
1999                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2000                                 fieldtype = fi->type;
2001                                 disp      = fi->offset;
2002                         }
2003
2004                         /* implicit null-pointer check */
2005                         switch (fieldtype) {
2006                         case TYPE_INT:
2007                         case TYPE_FLT:
2008                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2009                                 break;
2010                         case TYPE_LNG:
2011                         case TYPE_ADR:
2012                         case TYPE_DBL:
2013                                 /* XXX why no check for IS_IMM32? -- probably because of the patcher */
2014                                 M_MOV_IMM(iptr->sx.s23.s2.constval, REG_ITMP2);
2015                                 if (disp)  /* resolved, disp can never be 0 */
2016                                         M_LST(REG_ITMP2, s1, disp);
2017                                 else       /* unresolved */
2018                                         M_LST32(REG_ITMP2, s1, disp);
2019                                 break;
2020                         }
2021                         break;
2022
2023
2024                 /* branch operations **************************************************/
2025
2026                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2027
2028                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2029                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2030
2031                         PROFILE_CYCLE_STOP;
2032
2033 #ifdef ENABLE_VERIFIER
2034                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2035                                 uc = iptr->sx.s23.s2.uc;
2036
2037                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2038                         }
2039 #endif /* ENABLE_VERIFIER */
2040
2041                         M_CALL_IMM(0);                            /* passing exception pc */
2042                         M_POP(REG_ITMP2_XPC);
2043
2044                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2045                         M_JMP(REG_ITMP3);
2046                         break;
2047
2048                 case ICMD_GOTO:         /* ... ==> ...                                */
2049                 case ICMD_RET:
2050
2051                         emit_br(cd, iptr->dst.block);
2052                         ALIGNCODENOP;
2053                         break;
2054
2055                 case ICMD_JSR:          /* ... ==> ...                                */
2056
2057                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2058                         ALIGNCODENOP;
2059                         break;
2060
2061                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2062                 case ICMD_IFNONNULL:
2063
2064                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2065                         M_TEST(s1);
2066                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2067                         break;
2068
2069                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2070                 case ICMD_IFLT:
2071                 case ICMD_IFLE:
2072                 case ICMD_IFNE:
2073                 case ICMD_IFGT:
2074                 case ICMD_IFGE:
2075
2076                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2077                         M_ICMP_IMM(iptr->sx.val.i, s1);
2078                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2079                         break;
2080
2081                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2082                 case ICMD_IF_LNE:
2083                 case ICMD_IF_LLT:
2084                 case ICMD_IF_LGE:
2085                 case ICMD_IF_LGT:
2086                 case ICMD_IF_LLE:
2087
2088                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2089                         if (IS_IMM32(iptr->sx.val.l))
2090                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2091                         else {
2092                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2093                                 M_LCMP(REG_ITMP2, s1);
2094                         }
2095                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2096                         break;
2097
2098                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2099                 case ICMD_IF_ICMPNE:
2100                 case ICMD_IF_ICMPLT:
2101                 case ICMD_IF_ICMPGE:
2102                 case ICMD_IF_ICMPGT:
2103                 case ICMD_IF_ICMPLE:
2104
2105                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2106                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2107                         M_ICMP(s2, s1);
2108                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2109                         break;
2110
2111                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2112                 case ICMD_IF_ACMPNE:
2113
2114                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2115                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2116                         M_LCMP(s2, s1);
2117                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2118                         break;
2119
2120                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2121                 case ICMD_IF_LCMPNE:
2122                 case ICMD_IF_LCMPLT:
2123                 case ICMD_IF_LCMPGE:
2124                 case ICMD_IF_LCMPGT:
2125                 case ICMD_IF_LCMPLE:
2126
2127                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2128                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2129                         M_LCMP(s2, s1);
2130                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2131                         break;
2132
2133                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2134                 case ICMD_LRETURN:
2135
2136                         REPLACEMENT_POINT_RETURN(cd, iptr);
2137                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2138                         M_INTMOVE(s1, REG_RESULT);
2139                         goto nowperformreturn;
2140
2141                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2142
2143                         REPLACEMENT_POINT_RETURN(cd, iptr);
2144                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2145                         M_INTMOVE(s1, REG_RESULT);
2146
2147 #ifdef ENABLE_VERIFIER
2148                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2149                                 uc = iptr->sx.s23.s2.uc;
2150
2151                                 PROFILE_CYCLE_STOP;
2152
2153                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2154
2155                                 PROFILE_CYCLE_START;
2156                         }
2157 #endif /* ENABLE_VERIFIER */
2158                         goto nowperformreturn;
2159
2160                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2161                 case ICMD_DRETURN:
2162
2163                         REPLACEMENT_POINT_RETURN(cd, iptr);
2164                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2165                         M_FLTMOVE(s1, REG_FRESULT);
2166                         goto nowperformreturn;
2167
2168                 case ICMD_RETURN:      /* ...  ==> ...                                */
2169
2170                         REPLACEMENT_POINT_RETURN(cd, iptr);
2171
2172 nowperformreturn:
2173                         {
2174                         s4 i, p;
2175
2176                         p = cd->stackframesize;
2177
2178 #if !defined(NDEBUG)
2179                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2180                                 emit_verbosecall_exit(jd);
2181 #endif /* !defined(NDEBUG) */
2182
2183 #if defined(ENABLE_THREADS)
2184                         if (checksync && code_is_synchronized(code)) {
2185                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2186
2187                                 /* we need to save the proper return value */
2188                                 switch (iptr->opc) {
2189                                 case ICMD_IRETURN:
2190                                 case ICMD_ARETURN:
2191                                 case ICMD_LRETURN:
2192                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2193                                         break;
2194                                 case ICMD_FRETURN:
2195                                 case ICMD_DRETURN:
2196                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2197                                         break;
2198                                 }
2199
2200                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2201                                 M_CALL(REG_ITMP1);
2202
2203                                 /* and now restore the proper return value */
2204                                 switch (iptr->opc) {
2205                                 case ICMD_IRETURN:
2206                                 case ICMD_ARETURN:
2207                                 case ICMD_LRETURN:
2208                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2209                                         break;
2210                                 case ICMD_FRETURN:
2211                                 case ICMD_DRETURN:
2212                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2213                                         break;
2214                                 }
2215                         }
2216 #endif
2217
2218                         /* restore saved registers */
2219
2220                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2221                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2222                         }
2223                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2224                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2225                         }
2226
2227                         /* deallocate stack */
2228
2229                         if (cd->stackframesize)
2230                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2231
2232                         /* generate method profiling code */
2233
2234                         PROFILE_CYCLE_STOP;
2235
2236                         M_RET;
2237                         }
2238                         break;
2239
2240
2241                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2242                         {
2243                                 s4 i, l;
2244                                 branch_target_t *table;
2245
2246                                 table = iptr->dst.table;
2247
2248                                 l = iptr->sx.s23.s2.tablelow;
2249                                 i = iptr->sx.s23.s3.tablehigh;
2250
2251                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2252                                 M_INTMOVE(s1, REG_ITMP1);
2253
2254                                 if (l != 0)
2255                                         M_ISUB_IMM(l, REG_ITMP1);
2256
2257                                 /* number of targets */
2258                                 i = i - l + 1;
2259
2260                 /* range check */
2261
2262                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2263                                 emit_bugt(cd, table[0].block);
2264
2265                                 /* build jump table top down and use address of lowest entry */
2266
2267                                 table += i;
2268
2269                                 while (--i >= 0) {
2270                                         dseg_add_target(cd, table->block);
2271                                         --table;
2272                                 }
2273
2274                                 /* length of dataseg after last dseg_add_target is used
2275                                    by load */
2276
2277                                 M_MOV_IMM(0, REG_ITMP2);
2278                                 dseg_adddata(cd);
2279                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2280                                 M_JMP(REG_ITMP1);
2281                         }
2282                         break;
2283
2284
2285                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2286                         {
2287                                 s4 i;
2288                                 lookup_target_t *lookup;
2289
2290                                 lookup = iptr->dst.lookup;
2291
2292                                 i = iptr->sx.s23.s2.lookupcount;
2293
2294                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2295                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2296
2297                                 while (--i >= 0) {
2298                                         M_ICMP_IMM(lookup->value, s1);
2299                                         emit_beq(cd, lookup->target.block);
2300                                         lookup++;
2301                                 }
2302
2303                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2304                                 ALIGNCODENOP;
2305                         }
2306                         break;
2307
2308
2309                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2310
2311                         REPLACEMENT_POINT_FORGC_BUILTIN(cd, iptr);
2312
2313                         bte = iptr->sx.s23.s3.bte;
2314                         md  = bte->md;
2315                         goto gen_method;
2316
2317                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2318
2319                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2320                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2321                 case ICMD_INVOKEINTERFACE:
2322
2323                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2324
2325                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2326                                 lm = NULL;
2327                                 um = iptr->sx.s23.s3.um;
2328                                 md = um->methodref->parseddesc.md;
2329                         }
2330                         else {
2331                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2332                                 um = NULL;
2333                                 md = lm->parseddesc;
2334                         }
2335
2336 gen_method:
2337                         s3 = md->paramcount;
2338
2339                         MCODECHECK((20 * s3) + 128);
2340
2341                         /* copy arguments to registers or stack location */
2342
2343                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2344                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2345                                 d   = md->params[s3].regoff;
2346
2347                                 /* already preallocated (ARGVAR)? */
2348
2349                                 if (var->flags & PREALLOC)
2350                                         continue;
2351
2352                                 if (IS_INT_LNG_TYPE(var->type)) {
2353                                         if (!md->params[s3].inmemory) {
2354                                                 s1 = emit_load(jd, iptr, var, d);
2355                                                 M_INTMOVE(s1, d);
2356                                         }
2357                                         else {
2358                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2359                                                 M_LST(s1, REG_SP, d);
2360                                         }
2361                                 }
2362                                 else {
2363                                         if (!md->params[s3].inmemory) {
2364                                                 s1 = emit_load(jd, iptr, var, d);
2365                                                 M_FLTMOVE(s1, d);
2366                                         }
2367                                         else {
2368                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2369
2370                                                 if (IS_2_WORD_TYPE(var->type))
2371                                                         M_DST(s1, REG_SP, d);
2372                                                 else
2373                                                         M_FST(s1, REG_SP, d);
2374                                         }
2375                                 }
2376                         }
2377
2378                         /* generate method profiling code */
2379
2380                         PROFILE_CYCLE_STOP;
2381
2382                         switch (iptr->opc) {
2383                         case ICMD_BUILTIN:
2384                                 if (bte->stub == NULL) {
2385                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2386                                 }
2387                                 else {
2388                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2389                                 }
2390                                 M_CALL(REG_ITMP1);
2391                                 break;
2392
2393                         case ICMD_INVOKESPECIAL:
2394                                 emit_nullpointer_check(cd, iptr, REG_A0);
2395                                 /* fall through */
2396
2397                         case ICMD_INVOKESTATIC:
2398                                 if (lm == NULL) {
2399                                         disp = dseg_add_unique_address(cd, um);
2400
2401                                         patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
2402                                                                                   um, disp);
2403                                 }
2404                                 else {
2405                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2406                                 }
2407
2408                                 M_ALD(REG_ITMP2, RIP, disp);
2409                                 M_CALL(REG_ITMP2);
2410                                 break;
2411
2412                         case ICMD_INVOKEVIRTUAL:
2413                                 if (lm == NULL) {
2414                                         patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
2415
2416                                         s1 = 0;
2417                                 }
2418                                 else {
2419                                         s1 = OFFSET(vftbl_t, table[0]) +
2420                                                 sizeof(methodptr) * lm->vftblindex;
2421                                 }
2422
2423                                 /* implicit null-pointer check */
2424                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2425                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2426                                 M_CALL(REG_ITMP3);
2427                                 break;
2428
2429                         case ICMD_INVOKEINTERFACE:
2430                                 if (lm == NULL) {
2431                                         patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
2432
2433                                         s1 = 0;
2434                                         s2 = 0;
2435                                 }
2436                                 else {
2437                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2438                                                 sizeof(methodptr) * lm->clazz->index;
2439
2440                                         s2 = sizeof(methodptr) * (lm - lm->clazz->methods);
2441                                 }
2442
2443                                 /* implicit null-pointer check */
2444                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2445                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2446                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2447                                 M_CALL(REG_ITMP3);
2448                                 break;
2449                         }
2450
2451                         /* generate method profiling code */
2452
2453                         PROFILE_CYCLE_START;
2454
2455                         /* store size of call code in replacement point */
2456
2457                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2458                         REPLACEMENT_POINT_FORGC_BUILTIN_RETURN(cd, iptr);
2459
2460                         /* store return value */
2461
2462                         switch (md->returntype.type) {
2463                         case TYPE_INT:
2464                         case TYPE_LNG:
2465                         case TYPE_ADR:
2466                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2467                                 M_INTMOVE(REG_RESULT, s1);
2468                                 emit_store_dst(jd, iptr, s1);
2469                                 break;
2470                         case TYPE_FLT:
2471                         case TYPE_DBL:
2472                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2473                                 M_FLTMOVE(REG_FRESULT, s1);
2474                                 emit_store_dst(jd, iptr, s1);
2475                                 break;
2476                         default:
2477                                 /* TYPE_VOID */
2478                                 break;
2479                         }
2480                         break;
2481
2482
2483                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2484
2485                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2486                                 /* object type cast-check */
2487
2488                                 classinfo *super;
2489                                 s4         superindex;
2490
2491                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2492                                         super      = NULL;
2493                                         superindex = 0;
2494                                 }
2495                                 else {
2496                                         super      = iptr->sx.s23.s3.c.cls;
2497                                         superindex = super->index;
2498                                 }
2499
2500                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2501
2502                                 /* if class is not resolved, check which code to call */
2503
2504                                 if (super == NULL) {
2505                                         M_TEST(s1);
2506                                         emit_label_beq(cd, BRANCH_LABEL_1);
2507
2508                                         patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2509                                                                                   iptr->sx.s23.s3.c.ref, 0);
2510
2511                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2512                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2513                                         emit_label_beq(cd, BRANCH_LABEL_2);
2514                                 }
2515
2516                                 /* interface checkcast code */
2517
2518                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2519                                         if (super != NULL) {
2520                                                 M_TEST(s1);
2521                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2522                                         }
2523
2524                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2525
2526                                         if (super == NULL) {
2527                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
2528                                                                                           iptr->sx.s23.s3.c.ref,
2529                                                                                           0);
2530                                         }
2531
2532                                         M_ILD32(REG_ITMP3,
2533                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2534                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2535                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2536
2537                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2538                                                         OFFSET(vftbl_t, interfacetable[0]) -
2539                                                         superindex * sizeof(methodptr*));
2540                                         M_TEST(REG_ITMP3);
2541                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2542
2543                                         if (super == NULL)
2544                                                 emit_label_br(cd, BRANCH_LABEL_4);
2545                                         else
2546                                                 emit_label(cd, BRANCH_LABEL_3);
2547                                 }
2548
2549                                 /* class checkcast code */
2550
2551                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2552                                         if (super == NULL) {
2553                                                 emit_label(cd, BRANCH_LABEL_2);
2554
2555                                                 cr   = iptr->sx.s23.s3.c.ref;
2556                                                 disp = dseg_add_unique_address(cd, cr);
2557
2558                                                 patcher_add_patch_ref(jd,
2559                                                                                           PATCHER_resolve_classref_to_vftbl,
2560                                                                                           cr, disp);
2561                                         }
2562                                         else {
2563                                                 M_TEST(s1);
2564                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2565
2566                                                 disp = dseg_add_address(cd, super->vftbl);
2567                                         }
2568
2569                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2570                                         M_ALD(REG_ITMP3, RIP, disp);
2571
2572                                         M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2573
2574                                         /*                                      if (s1 != REG_ITMP1) { */
2575                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2576                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2577                                         /*                                                                                              REG_ITMP1); */
2578                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2579                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2580                                         /*                                                                                              REG_ITMP3); */
2581                                         /*  #if defined(ENABLE_THREADS) */
2582                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2583                                         /*  #endif */
2584                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2585
2586                                         /*                                      } else { */
2587
2588                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
2589                                         M_ISUB(REG_ITMP3, REG_ITMP2);
2590                                         M_ALD(REG_ITMP3, RIP, disp);
2591                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2592                                         /*                                      } */
2593
2594                                         M_ICMP(REG_ITMP3, REG_ITMP2);
2595                                         emit_classcast_check(cd, iptr, BRANCH_UGT, REG_ITMP3, s1);
2596
2597                                         if (super != NULL)
2598                                                 emit_label(cd, BRANCH_LABEL_5);
2599                                 }
2600
2601                                 if (super == NULL) {
2602                                         emit_label(cd, BRANCH_LABEL_1);
2603                                         emit_label(cd, BRANCH_LABEL_4);
2604                                 }
2605
2606                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2607                         }
2608                         else {
2609                                 /* array type cast-check */
2610
2611                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2612                                 M_INTMOVE(s1, REG_A0);
2613
2614                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2615                                         cr   = iptr->sx.s23.s3.c.ref;
2616                                         disp = dseg_add_unique_address(cd, cr);
2617
2618                                         patcher_add_patch_ref(jd,
2619                                                                                   PATCHER_resolve_classref_to_classinfo,
2620                                                                                   cr, disp);
2621                                 }
2622                                 else {
2623                                         disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2624                                 }
2625
2626                                 M_ALD(REG_A1, RIP, disp);
2627                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2628                                 M_CALL(REG_ITMP1);
2629
2630                                 /* s1 may have been destroyed over the function call */
2631                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2632                                 M_TEST(REG_RESULT);
2633                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2634
2635                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2636                         }
2637
2638                         M_INTMOVE(s1, d);
2639                         emit_store_dst(jd, iptr, d);
2640                         break;
2641
2642                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2643
2644                         {
2645                         classinfo *super;
2646                         s4         superindex;
2647
2648                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2649                                 super      = NULL;
2650                                 superindex = 0;
2651                         }
2652                         else {
2653                                 super      = iptr->sx.s23.s3.c.cls;
2654                                 superindex = super->index;
2655                         }
2656
2657                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2658                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2659
2660                         if (s1 == d) {
2661                                 M_INTMOVE(s1, REG_ITMP1);
2662                                 s1 = REG_ITMP1;
2663                         }
2664
2665                         M_CLR(d);
2666
2667                         /* if class is not resolved, check which code to call */
2668
2669                         if (super == NULL) {
2670                                 M_TEST(s1);
2671                                 emit_label_beq(cd, BRANCH_LABEL_1);
2672
2673                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2674                                                                           iptr->sx.s23.s3.c.ref, 0);
2675
2676                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2677                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2678                                 emit_label_beq(cd, BRANCH_LABEL_2);
2679                         }
2680
2681                         /* interface instanceof code */
2682
2683                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2684                                 if (super != NULL) {
2685                                         M_TEST(s1);
2686                                         emit_label_beq(cd, BRANCH_LABEL_3);
2687                                 }
2688
2689                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2690
2691                                 if (super == NULL) {
2692                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
2693                                                                                   iptr->sx.s23.s3.c.ref, 0);
2694                                 }
2695
2696                                 M_ILD32(REG_ITMP3,
2697                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2698                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2699
2700                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2701
2702                                 M_BLE(a);
2703                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2704                                                 OFFSET(vftbl_t, interfacetable[0]) -
2705                                                 superindex * sizeof(methodptr*));
2706                                 M_TEST(REG_ITMP1);
2707                                 M_SETNE(d);
2708
2709                                 if (super == NULL)
2710                                         emit_label_br(cd, BRANCH_LABEL_4);
2711                                 else
2712                                         emit_label(cd, BRANCH_LABEL_3);
2713                         }
2714
2715                         /* class instanceof code */
2716
2717                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2718                                 if (super == NULL) {
2719                                         emit_label(cd, BRANCH_LABEL_2);
2720
2721                                         cr   = iptr->sx.s23.s3.c.ref;
2722                                         disp = dseg_add_unique_address(cd, cr);
2723
2724                                         patcher_add_patch_ref(jd,
2725                                                                                   PATCHER_resolve_classref_to_vftbl,
2726                                                                                   cr, disp);
2727                                 }
2728                                 else {
2729                                         M_TEST(s1);
2730                                         emit_label_beq(cd, BRANCH_LABEL_5);
2731
2732                                         disp = dseg_add_address(cd, super->vftbl);
2733                                 }
2734
2735                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2736                                 M_ALD(REG_ITMP2, RIP, disp);
2737
2738                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
2739                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2740                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2741
2742                                 M_ISUB(REG_ITMP2, REG_ITMP1);
2743                                 M_CLR(d); /* may be REG_ITMP2 */
2744                                 M_ICMP(REG_ITMP3, REG_ITMP1);
2745                                 M_SETULE(d);
2746
2747                                 if (super != NULL)
2748                                         emit_label(cd, BRANCH_LABEL_5);
2749                         }
2750
2751                         if (super == NULL) {
2752                                 emit_label(cd, BRANCH_LABEL_1);
2753                                 emit_label(cd, BRANCH_LABEL_4);
2754                         }
2755
2756                         emit_store_dst(jd, iptr, d);
2757                         }
2758                         break;
2759
2760                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2761
2762                         /* check for negative sizes and copy sizes to stack if necessary  */
2763
2764                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2765
2766                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2767
2768                                 /* copy SAVEDVAR sizes to stack */
2769                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2770
2771                                 /* Already Preallocated? */
2772                                 if (!(var->flags & PREALLOC)) {
2773                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2774                                         M_LST(s2, REG_SP, s1 * 8);
2775                                 }
2776                         }
2777
2778                         /* a0 = dimension count */
2779
2780                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2781
2782                         /* is a patcher function set? */
2783
2784                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2785                                 cr   = iptr->sx.s23.s3.c.ref;
2786                                 disp = dseg_add_unique_address(cd, cr);
2787
2788                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
2789                                                                           cr, disp);
2790                         }
2791                         else {
2792                                 disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2793                         }
2794
2795                         /* a1 = classinfo */
2796
2797                         M_ALD(REG_A1, RIP, disp);
2798
2799                         /* a2 = pointer to dimensions = stack pointer */
2800
2801                         M_MOV(REG_SP, REG_A2);
2802
2803                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2804                         M_CALL(REG_ITMP1);
2805
2806                         /* check for exception before result assignment */
2807
2808                         emit_exception_check(cd, iptr);
2809
2810                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2811                         M_INTMOVE(REG_RESULT, s1);
2812                         emit_store_dst(jd, iptr, s1);
2813                         break;
2814
2815                 default:
2816                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2817                                                                                    iptr->opc);
2818                         return false;
2819         } /* switch */
2820
2821         } /* for instruction */
2822
2823         MCODECHECK(512); /* XXX require a lower number? */
2824
2825         /* At the end of a basic block we may have to append some nops,
2826            because the patcher stub calling code might be longer than the
2827            actual instruction. So codepatching does not change the
2828            following block unintentionally. */
2829
2830         if (cd->mcodeptr < cd->lastmcodeptr) {
2831                 while (cd->mcodeptr < cd->lastmcodeptr) {
2832                         M_NOP;
2833                 }
2834         }
2835
2836         } /* if (bptr -> flags >= BBREACHED) */
2837         } /* for basic block */
2838
2839         /* Generate patcher traps. */
2840
2841         emit_patcher_traps(jd);
2842
2843         /* everything's ok */
2844
2845         return true;
2846 }
2847
2848
2849 /* codegen_emit_stub_native ****************************************************
2850
2851    Emits a stub routine which calls a native method.
2852
2853 *******************************************************************************/
2854
2855 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f, int skipparams)
2856 {
2857         methodinfo  *m;
2858         codeinfo    *code;
2859         codegendata *cd;
2860         methoddesc  *md;
2861         int          i, j;
2862         int          s1, s2;
2863         int          disp;
2864
2865         /* Sanity check. */
2866
2867         assert(f != NULL);
2868
2869         /* Get required compiler data. */
2870
2871         m    = jd->m;
2872         code = jd->code;
2873         cd   = jd->cd;
2874
2875         /* initialize variables */
2876
2877         md = m->parseddesc;
2878
2879         /* calculate stack frame size */
2880
2881         cd->stackframesize =
2882                 sizeof(stackframeinfo_t) / SIZEOF_VOID_P +
2883                 sizeof(localref_table) / SIZEOF_VOID_P +
2884                 md->paramcount +
2885                 (md->returntype.type == TYPE_VOID ? 0 : 1) +
2886                 nmd->memuse;
2887
2888         ALIGN_ODD(cd->stackframesize);              /* keep stack 16-byte aligned */
2889
2890         /* create method header */
2891
2892         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2893         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2894         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2895         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2896         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2897
2898 #if defined(ENABLE_PROFILING)
2899         /* generate native method profiling code */
2900
2901         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
2902                 /* count frequency */
2903
2904                 M_MOV_IMM(code, REG_ITMP3);
2905                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
2906         }
2907 #endif
2908
2909         /* generate stub code */
2910
2911         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2912
2913 #if defined(ENABLE_GC_CACAO)
2914         /* Save callee saved integer registers in stackframeinfo (GC may
2915            need to recover them during a collection). */
2916
2917         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
2918                 OFFSET(stackframeinfo_t, intregs);
2919
2920         for (i = 0; i < INT_SAV_CNT; i++)
2921                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2922 #endif
2923
2924         /* save integer and float argument registers */
2925
2926         for (i = 0; i < md->paramcount; i++) {
2927                 if (!md->params[i].inmemory) {
2928                         s1 = md->params[i].regoff;
2929
2930                         switch (md->paramtypes[i].type) {
2931                         case TYPE_INT:
2932                         case TYPE_LNG:
2933                         case TYPE_ADR:
2934                                 M_LST(s1, REG_SP, i * 8);
2935                                 break;
2936                         case TYPE_FLT:
2937                         case TYPE_DBL:
2938                                 M_DST(s1, REG_SP, i * 8);
2939                                 break;
2940                         }
2941                 }
2942         }
2943
2944         /* create dynamic stack info */
2945
2946         M_MOV(REG_SP, REG_A0);
2947         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
2948         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
2949         M_CALL(REG_ITMP1);
2950
2951         /* remember class argument */
2952
2953         if (m->flags & ACC_STATIC)
2954                 M_MOV(REG_RESULT, REG_ITMP2);
2955
2956         /* restore integer and float argument registers */
2957
2958         for (i = 0; i < md->paramcount; i++) {
2959                 if (!md->params[i].inmemory) {
2960                         s1 = md->params[i].regoff;
2961
2962                         switch (md->paramtypes[i].type) {
2963                         case TYPE_INT:
2964                         case TYPE_LNG:
2965                         case TYPE_ADR:
2966                                 M_LLD(s1, REG_SP, i * 8);
2967                                 break;
2968                         case TYPE_FLT:
2969                         case TYPE_DBL:
2970                                 M_DLD(s1, REG_SP, i * 8);
2971                                 break;
2972                         }
2973                 }
2974         }
2975
2976         /* Copy or spill arguments to new locations. */
2977
2978         for (i = md->paramcount - 1, j = i + skipparams; i >= 0; i--, j--) {
2979                 s2 = nmd->params[j].regoff;
2980
2981                 switch (md->paramtypes[i].type) {
2982                 case TYPE_INT:
2983                 case TYPE_LNG:
2984                 case TYPE_ADR:
2985                         if (!md->params[i].inmemory) {
2986                                 s1 = md->params[i].regoff;
2987
2988                                 if (!nmd->params[j].inmemory)
2989                                         M_INTMOVE(s1, s2);
2990                                 else
2991                                         M_LST(s1, REG_SP, s2);
2992                         }
2993                         else {
2994                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
2995                                 M_LLD(REG_ITMP1, REG_SP, s1);
2996                                 M_LST(REG_ITMP1, REG_SP, s2);
2997                         }
2998                         break;
2999                 case TYPE_FLT:
3000                         /* We only copy spilled float arguments, as the float
3001                            argument registers keep unchanged. */
3002
3003                         if (md->params[i].inmemory) {
3004                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3005
3006                                 M_FLD(REG_FTMP1, REG_SP, s1);
3007                                 M_FST(REG_FTMP1, REG_SP, s2);
3008                         }
3009                         break;
3010                 case TYPE_DBL:
3011                         if (md->params[i].inmemory) {
3012                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3013                                 M_DLD(REG_FTMP1, REG_SP, s1);
3014                                 M_DST(REG_FTMP1, REG_SP, s2);
3015                         }
3016                         break;
3017                 }
3018         }
3019
3020         /* Handle native Java methods. */
3021
3022         if (m->flags & ACC_NATIVE) {
3023                 /* put class into second argument register */
3024
3025                 if (m->flags & ACC_STATIC)
3026                         M_MOV(REG_ITMP2, REG_A1);
3027
3028                 /* put env into first argument register */
3029
3030                 M_MOV_IMM(VM_get_jnienv(), REG_A0);
3031         }
3032
3033         /* Call the native function. */
3034
3035         disp = dseg_add_functionptr(cd, f);
3036         M_ALD(REG_ITMP1, RIP, disp);
3037         M_CALL(REG_ITMP1);
3038
3039         /* save return value */
3040
3041         switch (md->returntype.type) {
3042         case TYPE_INT:
3043         case TYPE_LNG:
3044         case TYPE_ADR:
3045                 switch (md->returntype.decltype) {
3046                 case PRIMITIVETYPE_BOOLEAN:
3047                         M_BZEXT(REG_RESULT, REG_RESULT);
3048                         break;
3049                 case PRIMITIVETYPE_BYTE:
3050                         M_BSEXT(REG_RESULT, REG_RESULT);
3051                         break;
3052                 case PRIMITIVETYPE_CHAR:
3053                         M_CZEXT(REG_RESULT, REG_RESULT);
3054                         break;
3055                 case PRIMITIVETYPE_SHORT:
3056                         M_SSEXT(REG_RESULT, REG_RESULT);
3057                         break;
3058                 }
3059                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3060                 break;
3061         case TYPE_FLT:
3062         case TYPE_DBL:
3063                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3064                 break;
3065         case TYPE_VOID:
3066                 break;
3067         }
3068
3069         /* remove native stackframe info */
3070
3071         M_MOV(REG_SP, REG_A0);
3072         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3073         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3074         M_CALL(REG_ITMP1);
3075         M_MOV(REG_RESULT, REG_ITMP3);
3076
3077         /* restore return value */
3078
3079         switch (md->returntype.type) {
3080         case TYPE_INT:
3081         case TYPE_LNG:
3082         case TYPE_ADR:
3083                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3084                 break;
3085         case TYPE_FLT:
3086         case TYPE_DBL:
3087                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3088                 break;
3089         case TYPE_VOID:
3090                 break;
3091         }
3092
3093 #if defined(ENABLE_GC_CACAO)
3094         /* Restore callee saved integer registers from stackframeinfo (GC
3095            might have modified them during a collection). */
3096
3097         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
3098                 OFFSET(stackframeinfo_t, intregs);
3099
3100         for (i = 0; i < INT_SAV_CNT; i++)
3101                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3102 #endif
3103
3104         /* remove stackframe */
3105
3106         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3107
3108         /* test for exception */
3109
3110         M_TEST(REG_ITMP3);
3111         M_BNE(1);
3112         M_RET;
3113
3114         /* handle exception */
3115
3116         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3117         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3118         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3119
3120         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3121         M_JMP(REG_ITMP3);
3122 }
3123
3124
3125 /*
3126  * These are local overrides for various environment variables in Emacs.
3127  * Please do not remove this and leave it at the end of the file, where
3128  * Emacs will automagically detect them.
3129  * ---------------------------------------------------------------------
3130  * Local variables:
3131  * mode: c
3132  * indent-tabs-mode: t
3133  * c-basic-offset: 4
3134  * tab-width: 4
3135  * End:
3136  * vim:noexpandtab:sw=4:ts=4:
3137  */