src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25 */
  26
  27
  28 #include "config.h"
  29
  30 #include <assert.h>
  31 #include <stdio.h>
  32 #include <stdint.h>
  33
  34 #include "vm/types.h"
  35
  36 #include "md-abi.h"
  37
  38 #include "vm/jit/x86_64/arch.h"
  39 #include "vm/jit/x86_64/codegen.h"
  40 #include "vm/jit/x86_64/emit.h"
  41
  42 #include "mm/memory.h"
  43
  44 #include "native/jni.h"
  45 #include "native/localref.h"
  46 #include "native/native.h"
  47
  48 #include "threads/lock-common.h"
  49
  50 #include "vm/builtin.h"
  51 #include "vm/exceptions.h"
  52 #include "vm/global.h"
  53 #include "vm/stringlocal.h"
  54 #include "vm/vm.h"
  55
  56 #include "vm/jit/abi.h"
  57 #include "vm/jit/asmpart.h"
  58 #include "vm/jit/code.h"
  59 #include "vm/jit/codegen-common.h"
  60 #include "vm/jit/dseg.h"
  61 #include "vm/jit/emit-common.h"
  62 #include "vm/jit/jit.h"
  63 #include "vm/jit/linenumbertable.h"
  64 #include "vm/jit/methodheader.h"
  65 #include "vm/jit/parse.h"
  66 #include "vm/jit/patcher-common.h"
  67 #include "vm/jit/reg.h"
  68 #include "vm/jit/replace.h"
  69 #include "vm/jit/stacktrace.h"
  70 #include "vm/jit/trap.h"
  71
  72 #if defined(ENABLE_LSRA)
  73 # include "vm/jit/allocator/lsra.h"
  74 #endif
  75
  76 #include "vmcore/loader.h"
  77 #include "vmcore/options.h"
  78 #include "vmcore/statistics.h"
  79
  80
  81 /* codegen_emit ****************************************************************
  82
  83    Generates machine code.
  84
  85 *******************************************************************************/
  86
  87 bool codegen_emit(jitdata *jd)
  88 {
  89         methodinfo         *m;
  90         codeinfo           *code;
  91         codegendata        *cd;
  92         registerdata       *rd;
  93         s4                  len, s1, s2, s3, d, disp;
  94         u2                  currentline;
  95         ptrint              a;
  96         varinfo            *var, *dst;
  97         basicblock         *bptr;
  98         instruction        *iptr;
  99         constant_classref  *cr;
 100         unresolved_class   *uc;
 101         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 102         unresolved_method  *um;
 103         builtintable_entry *bte;
 104         methoddesc         *md;
 105         fieldinfo          *fi;
 106         unresolved_field   *uf;
 107         s4                  fieldtype;
 108         s4                 varindex;
 109
 110         /* get required compiler data */
 111
 112         m    = jd->m;
 113         code = jd->code;
 114         cd   = jd->cd;
 115         rd   = jd->rd;
 116
 117         /* prevent compiler warnings */
 118
 119         d   = 0;
 120         lm  = NULL;
 121         um  = NULL;
 122         bte = NULL;
 123
 124         {
 125         s4 i, p, t, l;
 126         s4 savedregs_num;
 127
 128         savedregs_num = 0;
 129
 130         /* space to save used callee saved registers */
 131
 132         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 133         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 134
 135         cd->stackframesize = rd->memuse + savedregs_num;
 136
 137 #if defined(ENABLE_THREADS)
 138         /* space to save argument of monitor_enter */
 139
 140         if (checksync && code_is_synchronized(code))
 141                 cd->stackframesize++;
 142 #endif
 143
 144         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 145            native code e.g. libc or jni (alignment problems with
 146            movaps). */
 147
 148         if (!code_is_leafmethod(code) || opt_verbosecall)
 149                 cd->stackframesize |= 0x1;
 150
 151         /* create method header */
 152
 153         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 154         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 155
 156         code->synchronizedoffset = rd->memuse * 8;
 157
 158         if (code_is_leafmethod(code))
 159                 (void) dseg_add_unique_s4(cd, 1);                  /* IsLeaf          */
 160         else
 161                 (void) dseg_add_unique_s4(cd, 0);                  /* IsLeaf          */
 162
 163         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 164         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 165
 166 #if defined(ENABLE_PROFILING)
 167         /* generate method profiling code */
 168
 169         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 170                 /* count frequency */
 171
 172                 M_MOV_IMM(code, REG_ITMP3);
 173                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 174
 175                 PROFILE_CYCLE_START;
 176         }
 177 #endif
 178
 179         /* create stack frame (if necessary) */
 180
 181         if (cd->stackframesize)
 182                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 183
 184         /* save used callee saved registers */
 185
 186         p = cd->stackframesize;
 187         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 188                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 189         }
 190         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 191                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 192         }
 193
 194         /* take arguments out of register or stack frame */
 195
 196         md = m->parseddesc;
 197
 198         for (p = 0, l = 0; p < md->paramcount; p++) {
 199                 t = md->paramtypes[p].type;
 200
 201                 varindex = jd->local_map[l * 5 + t];
 202
 203                 l++;
 204                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 205                         l++;
 206
 207                 if (varindex == UNUSED)
 208                         continue;
 209
 210                 var = VAR(varindex);
 211
 212                 s1 = md->params[p].regoff;
 213
 214                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 215                         if (!md->params[p].inmemory) {           /* register arguments    */
 216                                 if (!IS_INMEMORY(var->flags))
 217                                         M_INTMOVE(s1, var->vv.regoff);
 218                                 else
 219                                     M_LST(s1, REG_SP, var->vv.regoff);
 220                         }
 221                         else {                                 /* stack arguments       */
 222                                 if (!IS_INMEMORY(var->flags))
 223                                         /* + 8 for return address */
 224                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 225                                 else
 226                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 227                         }
 228                 }
 229                 else {                                     /* floating args         */
 230                         if (!md->params[p].inmemory) {           /* register arguments    */
 231                                 if (!IS_INMEMORY(var->flags))
 232                                         M_FLTMOVE(s1, var->vv.regoff);
 233                                 else
 234                                         M_DST(s1, REG_SP, var->vv.regoff);
 235                         }
 236                         else {                                 /* stack arguments       */
 237                                 if (!IS_INMEMORY(var->flags))
 238                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 239                                 else
 240                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 241                         }
 242                 }
 243         }
 244
 245         /* save monitorenter argument */
 246
 247 #if defined(ENABLE_THREADS)
 248         if (checksync && code_is_synchronized(code)) {
 249                 /* stack offset for monitor argument */
 250
 251                 s1 = rd->memuse;
 252
 253                 if (opt_verbosecall) {
 254                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 255
 256                         for (p = 0; p < INT_ARG_CNT; p++)
 257                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 258
 259                         for (p = 0; p < FLT_ARG_CNT; p++)
 260                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 261
 262                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 263                 }
 264
 265                 /* decide which monitor enter function to call */
 266
 267                 if (m->flags & ACC_STATIC) {
 268                         M_MOV_IMM(&m->clazz->object.header, REG_A0);
 269                 }
 270                 else {
 271                         M_TEST(REG_A0);
 272                         M_BNE(8);
 273                         M_ALD_MEM(REG_A0, TRAP_NullPointerException);
 274                 }
 275
 276                 M_AST(REG_A0, REG_SP, s1 * 8);
 277                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 278                 M_CALL(REG_ITMP1);
 279
 280                 if (opt_verbosecall) {
 281                         for (p = 0; p < INT_ARG_CNT; p++)
 282                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 283
 284                         for (p = 0; p < FLT_ARG_CNT; p++)
 285                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 286
 287                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 288                 }
 289         }
 290 #endif
 291
 292 #if !defined(NDEBUG)
 293         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 294                 emit_verbosecall_enter(jd);
 295 #endif /* !defined(NDEBUG) */
 296
 297         }
 298
 299         /* end of header generation */
 300
 301         /* create replacement points */
 302
 303         REPLACEMENT_POINTS_INIT(cd, jd);
 304
 305         /* walk through all basic blocks */
 306
 307         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 308
 309                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 310
 311                 if (bptr->flags >= BBREACHED) {
 312
 313                 /* branch resolving */
 314
 315                 codegen_resolve_branchrefs(cd, bptr);
 316
 317                 /* handle replacement points */
 318
 319                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 320
 321                 /* copy interface registers to their destination */
 322
 323                 len = bptr->indepth;
 324                 MCODECHECK(512);
 325
 326 #if defined(ENABLE_PROFILING)
 327                 /* generate basicblock profiling code */
 328
 329                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 330                         /* count frequency */
 331
 332                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 333                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 334
 335                         /* if this is an exception handler, start profiling again */
 336
 337                         if (bptr->type == BBTYPE_EXH)
 338                                 PROFILE_CYCLE_START;
 339                 }
 340 #endif
 341
 342 #if defined(ENABLE_LSRA)
 343                 if (opt_lsra) {
 344                         while (len) {
 345                                 len--;
 346                                 src = bptr->invars[len];
 347                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 348                                         if (bptr->type == BBTYPE_EXH) {
 349 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 350                                                 if (!IS_INMEMORY(src->flags))
 351                                                         d= src->vv.regoff;
 352                                                 else
 353                                                         d=REG_ITMP1;
 354                                                 M_INTMOVE(REG_ITMP1, d);
 355                                                 emit_store(jd, NULL, src, d);
 356                                         }
 357                                 }
 358                         }
 359
 360                 } else {
 361 #endif
 362
 363                 while (len) {
 364                         len--;
 365                         var = VAR(bptr->invars[len]);
 366                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 367                                 if (bptr->type == BBTYPE_EXH) {
 368                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 369                                         M_INTMOVE(REG_ITMP1, d);
 370                                         emit_store(jd, NULL, var, d);
 371                                 }
 372                         }
 373                         else {
 374                                 assert((var->flags & INOUT));
 375                         }
 376                 }
 377 #if defined(ENABLE_LSRA)
 378                 }
 379 #endif
 380                 /* walk through all instructions */
 381
 382                 len = bptr->icount;
 383                 currentline = 0;
 384
 385                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 386                         if (iptr->line != currentline) {
 387                                 linenumbertable_list_entry_add(cd, iptr->line);
 388                                 currentline = iptr->line;
 389                         }
 390
 391                         MCODECHECK(1024);                         /* 1KB should be enough */
 392
 393                 switch (iptr->opc) {
 394                 case ICMD_NOP:        /* ...  ==> ...                                 */
 395                 case ICMD_POP:        /* ..., value  ==> ...                          */
 396                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 397                         break;
 398
 399                 case ICMD_INLINE_START:
 400
 401                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 402                         break;
 403
 404                 case ICMD_INLINE_BODY:
 405
 406                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 407                         linenumbertable_list_entry_add_inline_start(cd, iptr);
 408                         linenumbertable_list_entry_add(cd, iptr->line);
 409                         break;
 410
 411                 case ICMD_INLINE_END:
 412
 413                         linenumbertable_list_entry_add_inline_end(cd, iptr);
 414                         linenumbertable_list_entry_add(cd, iptr->line);
 415                         break;
 416
 417                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 418
 419                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 420                         emit_nullpointer_check(cd, iptr, s1);
 421                         break;
 422
 423                 /* constant operations ************************************************/
 424
 425                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 426
 427                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 428                         ICONST(d, iptr->sx.val.i);
 429                         emit_store_dst(jd, iptr, d);
 430                         break;
 431
 432                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 433
 434                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 435                         LCONST(d, iptr->sx.val.l);
 436                         emit_store_dst(jd, iptr, d);
 437                         break;
 438
 439                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 440
 441                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 442                         disp = dseg_add_float(cd, iptr->sx.val.f);
 443                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 444                         emit_store_dst(jd, iptr, d);
 445                         break;
 446
 447                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 448
 449                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 450                         disp = dseg_add_double(cd, iptr->sx.val.d);
 451                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 452                         emit_store_dst(jd, iptr, d);
 453                         break;
 454
 455                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 456
 457                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 458
 459                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 460                                 cr   = iptr->sx.val.c.ref;
 461                                 disp = dseg_add_unique_address(cd, cr);
 462
 463 /*                              PROFILE_CYCLE_STOP; */
 464
 465                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
 466                                                                           cr, disp);
 467
 468 /*                              PROFILE_CYCLE_START; */
 469
 470                                 M_ALD(d, RIP, disp);
 471                         }
 472                         else {
 473                                 if (iptr->sx.val.anyptr == 0) {
 474                                         M_CLR(d);
 475                                 }
 476                                 else {
 477                                         disp = dseg_add_address(cd, iptr->sx.val.anyptr);
 478                                         M_ALD(d, RIP, disp);
 479                                 }
 480                         }
 481                         emit_store_dst(jd, iptr, d);
 482                         break;
 483
 484
 485                 /* load/store/copy/move operations ************************************/
 486
 487                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 488                 case ICMD_ALOAD:      /* s1 = local variable                          */
 489                 case ICMD_LLOAD:
 490                 case ICMD_FLOAD:
 491                 case ICMD_DLOAD:
 492                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 493                 case ICMD_LSTORE:
 494                 case ICMD_FSTORE:
 495                 case ICMD_DSTORE:
 496                 case ICMD_COPY:
 497                 case ICMD_MOVE:
 498
 499                         emit_copy(jd, iptr);
 500                         break;
 501
 502                 case ICMD_ASTORE:
 503                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 504                                 emit_copy(jd, iptr);
 505                         break;
 506
 507                 /* integer operations *************************************************/
 508
 509                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 510
 511                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 512                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 513                         M_INTMOVE(s1, d);
 514                         M_INEG(d);
 515                         emit_store_dst(jd, iptr, d);
 516                         break;
 517
 518                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 519
 520                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 521                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 522                         M_INTMOVE(s1, d);
 523                         M_LNEG(d);
 524                         emit_store_dst(jd, iptr, d);
 525                         break;
 526
 527                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 528
 529                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 530                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 531                         M_ISEXT(s1, d);
 532                         emit_store_dst(jd, iptr, d);
 533                         break;
 534
 535                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 536
 537                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 538                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 539                         M_IMOV(s1, d);
 540                         emit_store_dst(jd, iptr, d);
 541                         break;
 542
 543                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 544
 545                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 546                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 547                         M_BSEXT(s1, d);
 548                         emit_store_dst(jd, iptr, d);
 549                         break;
 550
 551                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 552
 553                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 554                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 555                         M_CZEXT(s1, d);
 556                         emit_store_dst(jd, iptr, d);
 557                         break;
 558
 559                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 560
 561                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 562                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 563                         M_SSEXT(s1, d);
 564                         emit_store_dst(jd, iptr, d);
 565                         break;
 566
 567
 568                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 569
 570                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 571                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 572                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 573                         if (s2 == d)
 574                                 M_IADD(s1, d);
 575                         else {
 576                                 M_INTMOVE(s1, d);
 577                                 M_IADD(s2, d);
 578                         }
 579                         emit_store_dst(jd, iptr, d);
 580                         break;
 581
 582                 case ICMD_IINC:
 583                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 584                                       /* sx.val.i = constant                             */
 585
 586                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 587                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 588
 589                         /* Using inc and dec is not faster than add (tested with
 590                            sieve). */
 591
 592                         M_INTMOVE(s1, d);
 593                         M_IADD_IMM(iptr->sx.val.i, d);
 594                         emit_store_dst(jd, iptr, d);
 595                         break;
 596
 597                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 598
 599                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 600                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 601                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 602                         if (s2 == d)
 603                                 M_LADD(s1, d);
 604                         else {
 605                                 M_INTMOVE(s1, d);
 606                                 M_LADD(s2, d);
 607                         }
 608                         emit_store_dst(jd, iptr, d);
 609                         break;
 610
 611                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 612                                       /* sx.val.l = constant                             */
 613
 614                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 615                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 616                         M_INTMOVE(s1, d);
 617                         if (IS_IMM32(iptr->sx.val.l))
 618                                 M_LADD_IMM(iptr->sx.val.l, d);
 619                         else {
 620                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 621                                 M_LADD(REG_ITMP2, d);
 622                         }
 623                         emit_store_dst(jd, iptr, d);
 624                         break;
 625
 626                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 627
 628                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 629                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 630                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 631                         if (s2 == d) {
 632                                 M_INTMOVE(s1, REG_ITMP1);
 633                                 M_ISUB(s2, REG_ITMP1);
 634                                 M_INTMOVE(REG_ITMP1, d);
 635                         } else {
 636                                 M_INTMOVE(s1, d);
 637                                 M_ISUB(s2, d);
 638                         }
 639                         emit_store_dst(jd, iptr, d);
 640                         break;
 641
 642                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 643                                       /* sx.val.i = constant                             */
 644
 645                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 646                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 647                         M_INTMOVE(s1, d);
 648                         M_ISUB_IMM(iptr->sx.val.i, d);
 649                         emit_store_dst(jd, iptr, d);
 650                         break;
 651
 652                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 653
 654                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 655                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 656                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 657                         if (s2 == d) {
 658                                 M_INTMOVE(s1, REG_ITMP1);
 659                                 M_LSUB(s2, REG_ITMP1);
 660                                 M_INTMOVE(REG_ITMP1, d);
 661                         } else {
 662                                 M_INTMOVE(s1, d);
 663                                 M_LSUB(s2, d);
 664                         }
 665                         emit_store_dst(jd, iptr, d);
 666                         break;
 667
 668                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 669                                       /* sx.val.l = constant                             */
 670
 671                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 672                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 673                         M_INTMOVE(s1, d);
 674                         if (IS_IMM32(iptr->sx.val.l))
 675                                 M_LSUB_IMM(iptr->sx.val.l, d);
 676                         else {
 677                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 678                                 M_LSUB(REG_ITMP2, d);
 679                         }
 680                         emit_store_dst(jd, iptr, d);
 681                         break;
 682
 683                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 684
 685                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 686                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 687                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 688                         if (s2 == d)
 689                                 M_IMUL(s1, d);
 690                         else {
 691                                 M_INTMOVE(s1, d);
 692                                 M_IMUL(s2, d);
 693                         }
 694                         emit_store_dst(jd, iptr, d);
 695                         break;
 696
 697                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 698                                       /* sx.val.i = constant                             */
 699
 700                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 701                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 702                         if (iptr->sx.val.i == 2) {
 703                                 M_INTMOVE(s1, d);
 704                                 M_ISLL_IMM(1, d);
 705                         } else
 706                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 707                         emit_store_dst(jd, iptr, d);
 708                         break;
 709
 710                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 711
 712                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 713                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 714                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 715                         if (s2 == d)
 716                                 M_LMUL(s1, d);
 717                         else {
 718                                 M_INTMOVE(s1, d);
 719                                 M_LMUL(s2, d);
 720                         }
 721                         emit_store_dst(jd, iptr, d);
 722                         break;
 723
 724                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 725                                       /* sx.val.l = constant                             */
 726
 727                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 728                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 729                         if (IS_IMM32(iptr->sx.val.l))
 730                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 731                         else {
 732                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 733                                 M_INTMOVE(s1, d);
 734                                 M_LMUL(REG_ITMP2, d);
 735                         }
 736                         emit_store_dst(jd, iptr, d);
 737                         break;
 738
 739                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 740
 741                         s1 = emit_load_s1(jd, iptr, RAX);
 742                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 743                         d = codegen_reg_of_dst(jd, iptr, RAX);
 744
 745                         M_INTMOVE(s1, RAX);
 746                         M_INTMOVE(s2, REG_ITMP3);
 747                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 748
 749                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 750
 751                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 752                         M_BNE(4 + 6);
 753                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 754                         M_BEQ(1 + 3);                                          /* 6 bytes */
 755
 756                         emit_cltd(cd);                                         /* 1 byte  */
 757                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 758
 759                         M_INTMOVE(RAX, d);
 760                         emit_store_dst(jd, iptr, d);
 761                         dst = VAROP(iptr->dst);
 762                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 763                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 764                         break;
 765
 766                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 767
 768                         s1 = emit_load_s1(jd, iptr, RAX);
 769                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 770                         d = codegen_reg_of_dst(jd, iptr, RDX);
 771
 772                         M_INTMOVE(s1, RAX);
 773                         M_INTMOVE(s2, REG_ITMP3);
 774                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 775
 776                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 777
 778                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 779                         M_BNE(3 + 4 + 6);
 780                         M_CLR(RDX);                                            /* 3 bytes */
 781                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 782                         M_BEQ(1 + 3);                                          /* 6 bytes */
 783
 784                         emit_cltd(cd);                                         /* 1 byte  */
 785                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 786
 787                         M_INTMOVE(RDX, d);
 788                         emit_store_dst(jd, iptr, d);
 789                         dst = VAROP(iptr->dst);
 790                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 791                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 792                         break;
 793
 794                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 795                                       /* sx.val.i = constant                             */
 796
 797                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 798                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 799                         M_INTMOVE(s1, REG_ITMP1);
 800                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 801                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 802                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 803                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 804                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 805                         emit_store_dst(jd, iptr, d);
 806                         break;
 807
 808                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 809                                       /* sx.val.i = constant                             */
 810
 811                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 812                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 813                         M_INTMOVE(s1, REG_ITMP1);
 814                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 815                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 816                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 817                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 818                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 819                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 820                         emit_store_dst(jd, iptr, d);
 821                         break;
 822
 823
 824                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 825
 826                         s1 = emit_load_s1(jd, iptr, RAX);
 827                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 828                         d = codegen_reg_of_dst(jd, iptr, RAX);
 829
 830                         M_INTMOVE(s1, RAX);
 831                         M_INTMOVE(s2, REG_ITMP3);
 832                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 833
 834                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 835
 836                         /* check as described in jvm spec */
 837                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 838                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 839                         M_BNE(4 + 6);
 840                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 841                         M_BEQ(2 + 3);                                          /* 6 bytes */
 842
 843                         emit_cqto(cd);                                         /* 2 bytes */
 844                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 845
 846                         M_INTMOVE(RAX, d);
 847                         emit_store_dst(jd, iptr, d);
 848                         dst = VAROP(iptr->dst);
 849                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 850                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 851                         break;
 852
 853                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 854
 855                         s1 = emit_load_s1(jd, iptr, RAX);
 856                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 857                         d = codegen_reg_of_dst(jd, iptr, RDX);
 858
 859                         M_INTMOVE(s1, RAX);
 860                         M_INTMOVE(s2, REG_ITMP3);
 861                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 862
 863                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 864
 865                         /* check as described in jvm spec */
 866                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 867                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 868                         M_BNE(3 + 4 + 6);
 869                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 870                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 871                         M_BEQ(2 + 3);                                          /* 6 bytes */
 872
 873                         emit_cqto(cd);                                         /* 2 bytes */
 874                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 875
 876                         M_INTMOVE(RDX, d);
 877                         emit_store_dst(jd, iptr, d);
 878                         dst = VAROP(iptr->dst);
 879                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 880                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 881                         break;
 882
 883                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 884                                       /* sx.val.i = constant                             */
 885
 886                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 887                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 888                         M_INTMOVE(s1, REG_ITMP1);
 889                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 890                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 891                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 892                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 893                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 894                         emit_store_dst(jd, iptr, d);
 895                         break;
 896
 897                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 898                                       /* sx.val.l = constant                             */
 899
 900                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 901                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 902                         M_INTMOVE(s1, REG_ITMP1);
 903                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 904                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 905                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 906                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 907                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 908                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 909                         emit_store_dst(jd, iptr, d);
 910                         break;
 911
 912                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 913
 914                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 915                         emit_ishift(jd, SHIFT_SHL, iptr);
 916                         break;
 917
 918                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 919                                       /* sx.val.i = constant                             */
 920
 921                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 922                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 923                         M_INTMOVE(s1, d);
 924                         M_ISLL_IMM(iptr->sx.val.i, d);
 925                         emit_store_dst(jd, iptr, d);
 926                         break;
 927
 928                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 929
 930                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 931                         emit_ishift(jd, SHIFT_SAR, iptr);
 932                         break;
 933
 934                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 935                                       /* sx.val.i = constant                             */
 936
 937                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 938                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 939                         M_INTMOVE(s1, d);
 940                         M_ISRA_IMM(iptr->sx.val.i, d);
 941                         emit_store_dst(jd, iptr, d);
 942                         break;
 943
 944                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 945
 946                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 947                         emit_ishift(jd, SHIFT_SHR, iptr);
 948                         break;
 949
 950                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 951                                       /* sx.val.i = constant                             */
 952
 953                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 954                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 955                         M_INTMOVE(s1, d);
 956                         M_ISRL_IMM(iptr->sx.val.i, d);
 957                         emit_store_dst(jd, iptr, d);
 958                         break;
 959
 960                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 961
 962                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 963                         emit_lshift(jd, SHIFT_SHL, iptr);
 964                         break;
 965
 966         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 967                                           /* sx.val.i = constant                             */
 968
 969                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 970                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 971                         M_INTMOVE(s1, d);
 972                         M_LSLL_IMM(iptr->sx.val.i, d);
 973                         emit_store_dst(jd, iptr, d);
 974                         break;
 975
 976                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 977
 978                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 979                         emit_lshift(jd, SHIFT_SAR, iptr);
 980                         break;
 981
 982                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 983                                       /* sx.val.i = constant                             */
 984
 985                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 986                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 987                         M_INTMOVE(s1, d);
 988                         M_LSRA_IMM(iptr->sx.val.i, d);
 989                         emit_store_dst(jd, iptr, d);
 990                         break;
 991
 992                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 993
 994                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 995                         emit_lshift(jd, SHIFT_SHR, iptr);
 996                         break;
 997
 998                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 999                                       /* sx.val.l = constant                             */
1000
1001                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1002                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1003                         M_INTMOVE(s1, d);
1004                         M_LSRL_IMM(iptr->sx.val.i, d);
1005                         emit_store_dst(jd, iptr, d);
1006                         break;
1007
1008                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1009
1010                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1011                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1012                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1013                         if (s2 == d)
1014                                 M_IAND(s1, d);
1015                         else {
1016                                 M_INTMOVE(s1, d);
1017                                 M_IAND(s2, d);
1018                         }
1019                         emit_store_dst(jd, iptr, d);
1020                         break;
1021
1022                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1023                                       /* sx.val.i = constant                             */
1024
1025                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1026                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1027                         M_INTMOVE(s1, d);
1028                         M_IAND_IMM(iptr->sx.val.i, d);
1029                         emit_store_dst(jd, iptr, d);
1030                         break;
1031
1032                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1033
1034                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1035                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1036                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1037                         if (s2 == d)
1038                                 M_LAND(s1, d);
1039                         else {
1040                                 M_INTMOVE(s1, d);
1041                                 M_LAND(s2, d);
1042                         }
1043                         emit_store_dst(jd, iptr, d);
1044                         break;
1045
1046                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1047                                       /* sx.val.l = constant                             */
1048
1049                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1050                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1051                         M_INTMOVE(s1, d);
1052                         if (IS_IMM32(iptr->sx.val.l))
1053                                 M_LAND_IMM(iptr->sx.val.l, d);
1054                         else {
1055                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1056                                 M_LAND(REG_ITMP2, d);
1057                         }
1058                         emit_store_dst(jd, iptr, d);
1059                         break;
1060
1061                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1062
1063                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1064                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1065                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1066                         if (s2 == d)
1067                                 M_IOR(s1, d);
1068                         else {
1069                                 M_INTMOVE(s1, d);
1070                                 M_IOR(s2, d);
1071                         }
1072                         emit_store_dst(jd, iptr, d);
1073                         break;
1074
1075                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1076                                       /* sx.val.i = constant                             */
1077
1078                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1079                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1080                         M_INTMOVE(s1, d);
1081                         M_IOR_IMM(iptr->sx.val.i, d);
1082                         emit_store_dst(jd, iptr, d);
1083                         break;
1084
1085                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1086
1087                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1088                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1089                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1090                         if (s2 == d)
1091                                 M_LOR(s1, d);
1092                         else {
1093                                 M_INTMOVE(s1, d);
1094                                 M_LOR(s2, d);
1095                         }
1096                         emit_store_dst(jd, iptr, d);
1097                         break;
1098
1099                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1100                                       /* sx.val.l = constant                             */
1101
1102                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1103                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1104                         M_INTMOVE(s1, d);
1105                         if (IS_IMM32(iptr->sx.val.l))
1106                                 M_LOR_IMM(iptr->sx.val.l, d);
1107                         else {
1108                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1109                                 M_LOR(REG_ITMP2, d);
1110                         }
1111                         emit_store_dst(jd, iptr, d);
1112                         break;
1113
1114                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1115
1116                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1117                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1118                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1119                         if (s2 == d)
1120                                 M_IXOR(s1, d);
1121                         else {
1122                                 M_INTMOVE(s1, d);
1123                                 M_IXOR(s2, d);
1124                         }
1125                         emit_store_dst(jd, iptr, d);
1126                         break;
1127
1128                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1129                                       /* sx.val.i = constant                             */
1130
1131                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1132                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1133                         M_INTMOVE(s1, d);
1134                         M_IXOR_IMM(iptr->sx.val.i, d);
1135                         emit_store_dst(jd, iptr, d);
1136                         break;
1137
1138                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1139
1140                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1141                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1142                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1143                         if (s2 == d)
1144                                 M_LXOR(s1, d);
1145                         else {
1146                                 M_INTMOVE(s1, d);
1147                                 M_LXOR(s2, d);
1148                         }
1149                         emit_store_dst(jd, iptr, d);
1150                         break;
1151
1152                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1153                                       /* sx.val.l = constant                             */
1154
1155                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1156                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1157                         M_INTMOVE(s1, d);
1158                         if (IS_IMM32(iptr->sx.val.l))
1159                                 M_LXOR_IMM(iptr->sx.val.l, d);
1160                         else {
1161                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1162                                 M_LXOR(REG_ITMP2, d);
1163                         }
1164                         emit_store_dst(jd, iptr, d);
1165                         break;
1166
1167
1168                 /* floating operations ************************************************/
1169
1170                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1171
1172                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1173                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1174                         disp = dseg_add_s4(cd, 0x80000000);
1175                         M_FLTMOVE(s1, d);
1176                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1177                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1178                         emit_store_dst(jd, iptr, d);
1179                         break;
1180
1181                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1182
1183                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1184                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1185                         disp = dseg_add_s8(cd, 0x8000000000000000);
1186                         M_FLTMOVE(s1, d);
1187                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1188                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1189                         emit_store_dst(jd, iptr, d);
1190                         break;
1191
1192                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1193
1194                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1195                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1196                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1197                         if (s2 == d)
1198                                 M_FADD(s1, d);
1199                         else {
1200                                 M_FLTMOVE(s1, d);
1201                                 M_FADD(s2, d);
1202                         }
1203                         emit_store_dst(jd, iptr, d);
1204                         break;
1205
1206                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1207
1208                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1209                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1210                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1211                         if (s2 == d)
1212                                 M_DADD(s1, d);
1213                         else {
1214                                 M_FLTMOVE(s1, d);
1215                                 M_DADD(s2, d);
1216                         }
1217                         emit_store_dst(jd, iptr, d);
1218                         break;
1219
1220                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1221
1222                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1223                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1224                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1225                         if (s2 == d) {
1226                                 M_FLTMOVE(s2, REG_FTMP2);
1227                                 s2 = REG_FTMP2;
1228                         }
1229                         M_FLTMOVE(s1, d);
1230                         M_FSUB(s2, d);
1231                         emit_store_dst(jd, iptr, d);
1232                         break;
1233
1234                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1235
1236                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1237                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1238                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1239                         if (s2 == d) {
1240                                 M_FLTMOVE(s2, REG_FTMP2);
1241                                 s2 = REG_FTMP2;
1242                         }
1243                         M_FLTMOVE(s1, d);
1244                         M_DSUB(s2, d);
1245                         emit_store_dst(jd, iptr, d);
1246                         break;
1247
1248                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1249
1250                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1251                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1252                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1253                         if (s2 == d)
1254                                 M_FMUL(s1, d);
1255                         else {
1256                                 M_FLTMOVE(s1, d);
1257                                 M_FMUL(s2, d);
1258                         }
1259                         emit_store_dst(jd, iptr, d);
1260                         break;
1261
1262                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1263
1264                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1265                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1266                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1267                         if (s2 == d)
1268                                 M_DMUL(s1, d);
1269                         else {
1270                                 M_FLTMOVE(s1, d);
1271                                 M_DMUL(s2, d);
1272                         }
1273                         emit_store_dst(jd, iptr, d);
1274                         break;
1275
1276                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1277
1278                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1279                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1280                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1281                         if (s2 == d) {
1282                                 M_FLTMOVE(s2, REG_FTMP2);
1283                                 s2 = REG_FTMP2;
1284                         }
1285                         M_FLTMOVE(s1, d);
1286                         M_FDIV(s2, d);
1287                         emit_store_dst(jd, iptr, d);
1288                         break;
1289
1290                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1291
1292                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1293                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1294                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1295                         if (s2 == d) {
1296                                 M_FLTMOVE(s2, REG_FTMP2);
1297                                 s2 = REG_FTMP2;
1298                         }
1299                         M_FLTMOVE(s1, d);
1300                         M_DDIV(s2, d);
1301                         emit_store_dst(jd, iptr, d);
1302                         break;
1303
1304                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1305
1306                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1307                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1308                         M_CVTIF(s1, d);
1309                         emit_store_dst(jd, iptr, d);
1310                         break;
1311
1312                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1313
1314                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1315                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1316                         M_CVTID(s1, d);
1317                         emit_store_dst(jd, iptr, d);
1318                         break;
1319
1320                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1321
1322                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1323                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1324                         M_CVTLF(s1, d);
1325                         emit_store_dst(jd, iptr, d);
1326                         break;
1327
1328                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1329
1330                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1331                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1332                         M_CVTLD(s1, d);
1333                         emit_store_dst(jd, iptr, d);
1334                         break;
1335
1336                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1337
1338                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1339                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1340                         M_CVTFI(s1, d);
1341                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1342                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1343                                 ((REG_RESULT == d) ? 0 : 3);
1344                         M_BNE(disp);
1345                         M_FLTMOVE(s1, REG_FTMP1);
1346                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1347                         M_CALL(REG_ITMP2);
1348                         M_INTMOVE(REG_RESULT, d);
1349                         emit_store_dst(jd, iptr, d);
1350                         break;
1351
1352                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1353
1354                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1355                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1356                         M_CVTDI(s1, d);
1357                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1358                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1359                                 ((REG_RESULT == d) ? 0 : 3);
1360                         M_BNE(disp);
1361                         M_FLTMOVE(s1, REG_FTMP1);
1362                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1363                         M_CALL(REG_ITMP2);
1364                         M_INTMOVE(REG_RESULT, d);
1365                         emit_store_dst(jd, iptr, d);
1366                         break;
1367
1368                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1369
1370                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1371                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1372                         M_CVTFL(s1, d);
1373                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1374                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1375                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1376                                 ((REG_RESULT == d) ? 0 : 3);
1377                         M_BNE(disp);
1378                         M_FLTMOVE(s1, REG_FTMP1);
1379                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1380                         M_CALL(REG_ITMP2);
1381                         M_INTMOVE(REG_RESULT, d);
1382                         emit_store_dst(jd, iptr, d);
1383                         break;
1384
1385                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1386
1387                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1388                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1389                         M_CVTDL(s1, d);
1390                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1391                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1392                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1393                                 ((REG_RESULT == d) ? 0 : 3);
1394                         M_BNE(disp);
1395                         M_FLTMOVE(s1, REG_FTMP1);
1396                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1397                         M_CALL(REG_ITMP2);
1398                         M_INTMOVE(REG_RESULT, d);
1399                         emit_store_dst(jd, iptr, d);
1400                         break;
1401
1402                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1403
1404                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1405                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1406                         M_CVTFD(s1, d);
1407                         emit_store_dst(jd, iptr, d);
1408                         break;
1409
1410                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1411
1412                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1413                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1414                         M_CVTDF(s1, d);
1415                         emit_store_dst(jd, iptr, d);
1416                         break;
1417
1418                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1419                                           /* == => 0, < => 1, > => -1 */
1420
1421                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1422                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1423                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1424                         M_CLR(d);
1425                         M_MOV_IMM(1, REG_ITMP1);
1426                         M_MOV_IMM(-1, REG_ITMP2);
1427                         emit_ucomiss_reg_reg(cd, s1, s2);
1428                         M_CMOVULT(REG_ITMP1, d);
1429                         M_CMOVUGT(REG_ITMP2, d);
1430                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1431                         emit_store_dst(jd, iptr, d);
1432                         break;
1433
1434                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1435                                           /* == => 0, < => 1, > => -1 */
1436
1437                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1438                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1439                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1440                         M_CLR(d);
1441                         M_MOV_IMM(1, REG_ITMP1);
1442                         M_MOV_IMM(-1, REG_ITMP2);
1443                         emit_ucomiss_reg_reg(cd, s1, s2);
1444                         M_CMOVULT(REG_ITMP1, d);
1445                         M_CMOVUGT(REG_ITMP2, d);
1446                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1447                         emit_store_dst(jd, iptr, d);
1448                         break;
1449
1450                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1451                                           /* == => 0, < => 1, > => -1 */
1452
1453                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1454                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1455                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1456                         M_CLR(d);
1457                         M_MOV_IMM(1, REG_ITMP1);
1458                         M_MOV_IMM(-1, REG_ITMP2);
1459                         emit_ucomisd_reg_reg(cd, s1, s2);
1460                         M_CMOVULT(REG_ITMP1, d);
1461                         M_CMOVUGT(REG_ITMP2, d);
1462                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1463                         emit_store_dst(jd, iptr, d);
1464                         break;
1465
1466                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1467                                           /* == => 0, < => 1, > => -1 */
1468
1469                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1470                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1471                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1472                         M_CLR(d);
1473                         M_MOV_IMM(1, REG_ITMP1);
1474                         M_MOV_IMM(-1, REG_ITMP2);
1475                         emit_ucomisd_reg_reg(cd, s1, s2);
1476                         M_CMOVULT(REG_ITMP1, d);
1477                         M_CMOVUGT(REG_ITMP2, d);
1478                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1479                         emit_store_dst(jd, iptr, d);
1480                         break;
1481
1482
1483                 /* memory operations **************************************************/
1484
1485                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1486
1487                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1488                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1489                         /* implicit null-pointer check */
1490                         M_ILD(d, s1, OFFSET(java_array_t, size));
1491                         emit_store_dst(jd, iptr, d);
1492                         break;
1493
1494                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1495
1496                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1497                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1498                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1499                         /* implicit null-pointer check */
1500                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1501                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1502                         emit_store_dst(jd, iptr, d);
1503                         break;
1504
1505                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1506
1507                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1508                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1509                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1510                         /* implicit null-pointer check */
1511                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1512                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1513                         emit_store_dst(jd, iptr, d);
1514                         break;
1515
1516                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1517
1518                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1519                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1520                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1521                         /* implicit null-pointer check */
1522                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1523                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1524                         emit_store_dst(jd, iptr, d);
1525                         break;
1526
1527                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1528
1529                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1530                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1531                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1532                         /* implicit null-pointer check */
1533                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1534                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1535                         emit_store_dst(jd, iptr, d);
1536                         break;
1537
1538                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1539
1540                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1541                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1542                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1543                         /* implicit null-pointer check */
1544                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1545                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1546                         emit_store_dst(jd, iptr, d);
1547                         break;
1548
1549                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1550
1551                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1552                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1553                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1554                         /* implicit null-pointer check */
1555                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1556                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1557                         emit_store_dst(jd, iptr, d);
1558                         break;
1559
1560                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1561
1562                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1563                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1564                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1565                         /* implicit null-pointer check */
1566                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1567                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1568                         emit_store_dst(jd, iptr, d);
1569                         break;
1570
1571                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1572
1573                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1574                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1575                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1576                         /* implicit null-pointer check */
1577                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1578                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1579                         emit_store_dst(jd, iptr, d);
1580                         break;
1581
1582
1583                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1584
1585                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1586                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1587                         /* implicit null-pointer check */
1588                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1589                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1590                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1591                         break;
1592
1593                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1594
1595                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1596                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1597                         /* implicit null-pointer check */
1598                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1599                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1600                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1601                         break;
1602
1603                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1604
1605                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1606                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1607                         /* implicit null-pointer check */
1608                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1609                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1610                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1611                         break;
1612
1613                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1614
1615                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1616                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1617                         /* implicit null-pointer check */
1618                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1619                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1620                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1621                         break;
1622
1623                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1624
1625                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1626                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1627                         /* implicit null-pointer check */
1628                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1629                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1630                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1631                         break;
1632
1633                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1634
1635                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1636                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1637                         /* implicit null-pointer check */
1638                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1639                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1640                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1641                         break;
1642
1643                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1644
1645                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1646                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1647                         /* implicit null-pointer check */
1648                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1649                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1650                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1651                         break;
1652
1653                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1654
1655                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1656                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1657                         /* implicit null-pointer check */
1658                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1659                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1660
1661                         M_MOV(s1, REG_A0);
1662                         M_MOV(s3, REG_A1);
1663                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
1664                         M_CALL(REG_ITMP1);
1665                         emit_arraystore_check(cd, iptr);
1666
1667                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1668                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1669                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1670                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1671                         break;
1672
1673
1674                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1675
1676                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1677                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1678                         /* implicit null-pointer check */
1679                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1680                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1681                         break;
1682
1683                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1684
1685                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1686                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1687                         /* implicit null-pointer check */
1688                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1689                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1690                         break;
1691
1692                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1693
1694                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1695                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1696                         /* implicit null-pointer check */
1697                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1698                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1699                         break;
1700
1701                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1702
1703                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1704                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1705                         /* implicit null-pointer check */
1706                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1707                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1708                         break;
1709
1710                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1711
1712                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1713                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1714                         /* implicit null-pointer check */
1715                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1716
1717                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1718                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1719                         }
1720                         else {
1721                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1722                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1723                         }
1724                         break;
1725
1726                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1727
1728                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1729                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1730                         /* implicit null-pointer check */
1731                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1732                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1733                         break;
1734
1735
1736                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1737
1738                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1739                                 uf        = iptr->sx.s23.s3.uf;
1740                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1741                                 disp      = dseg_add_unique_address(cd, uf);
1742
1743 /*                              PROFILE_CYCLE_STOP; */
1744
1745                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1746
1747 /*                              PROFILE_CYCLE_START; */
1748                         }
1749                         else {
1750                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1751                                 fieldtype = fi->type;
1752                                 disp      = dseg_add_address(cd, fi->value);
1753
1754                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1755                                         PROFILE_CYCLE_STOP;
1756
1757                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1758                                                                                   fi->clazz, 0);
1759
1760                                         PROFILE_CYCLE_START;
1761                                 }
1762                         }
1763
1764                         /* This approach is much faster than moving the field
1765                            address inline into a register. */
1766
1767                         M_ALD(REG_ITMP1, RIP, disp);
1768
1769                         switch (fieldtype) {
1770                         case TYPE_INT:
1771                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1772                                 M_ILD(d, REG_ITMP1, 0);
1773                                 break;
1774                         case TYPE_LNG:
1775                         case TYPE_ADR:
1776                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1777                                 M_LLD(d, REG_ITMP1, 0);
1778                                 break;
1779                         case TYPE_FLT:
1780                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1781                                 M_FLD(d, REG_ITMP1, 0);
1782                                 break;
1783                         case TYPE_DBL:
1784                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1785                                 M_DLD(d, REG_ITMP1, 0);
1786                                 break;
1787                         }
1788                         emit_store_dst(jd, iptr, d);
1789                         break;
1790
1791                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1792
1793                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1794                                 uf        = iptr->sx.s23.s3.uf;
1795                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1796                                 disp      = dseg_add_unique_address(cd, uf);
1797
1798 /*                              PROFILE_CYCLE_STOP; */
1799
1800                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1801
1802 /*                              PROFILE_CYCLE_START; */
1803                         }
1804                         else {
1805                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1806                                 fieldtype = fi->type;
1807                                 disp      = dseg_add_address(cd, fi->value);
1808
1809                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1810                                         PROFILE_CYCLE_STOP;
1811
1812                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1813                                                                                   fi->clazz, 0);
1814
1815                                         PROFILE_CYCLE_START;
1816                                 }
1817                         }
1818
1819                         /* This approach is much faster than moving the field
1820                            address inline into a register. */
1821
1822                         M_ALD(REG_ITMP1, RIP, disp);
1823
1824                         switch (fieldtype) {
1825                         case TYPE_INT:
1826                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1827                                 M_IST(s1, REG_ITMP1, 0);
1828                                 break;
1829                         case TYPE_LNG:
1830                         case TYPE_ADR:
1831                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1832                                 M_LST(s1, REG_ITMP1, 0);
1833                                 break;
1834                         case TYPE_FLT:
1835                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1836                                 M_FST(s1, REG_ITMP1, 0);
1837                                 break;
1838                         case TYPE_DBL:
1839                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1840                                 M_DST(s1, REG_ITMP1, 0);
1841                                 break;
1842                         }
1843                         break;
1844
1845                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1846                                           /* val = value (in current instruction)     */
1847                                           /* following NOP)                           */
1848
1849                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1850                                 uf        = iptr->sx.s23.s3.uf;
1851                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1852                                 disp      = dseg_add_unique_address(cd, uf);
1853
1854 /*                              PROFILE_CYCLE_STOP; */
1855
1856                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1857
1858 /*                              PROFILE_CYCLE_START; */
1859                         }
1860                         else {
1861                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1862                                 fieldtype = fi->type;
1863                                 disp      = dseg_add_address(cd, fi->value);
1864
1865                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1866                                         PROFILE_CYCLE_STOP;
1867
1868                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1869                                                                                   fi->clazz, 0);
1870
1871                                         PROFILE_CYCLE_START;
1872                                 }
1873                         }
1874
1875                         /* This approach is much faster than moving the field
1876                            address inline into a register. */
1877
1878                         M_ALD(REG_ITMP1, RIP, disp);
1879
1880                         switch (fieldtype) {
1881                         case TYPE_INT:
1882                         case TYPE_FLT:
1883                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1884                                 break;
1885                         case TYPE_LNG:
1886                         case TYPE_ADR:
1887                         case TYPE_DBL:
1888                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1889                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1890                                 else {
1891                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1892                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1893                                 }
1894                                 break;
1895                         }
1896                         break;
1897
1898                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1899
1900                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1901
1902                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1903                                 uf        = iptr->sx.s23.s3.uf;
1904                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1905                                 disp      = 0;
1906
1907 /*                              PROFILE_CYCLE_STOP; */
1908
1909                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1910
1911 /*                              PROFILE_CYCLE_START; */
1912                         }
1913                         else {
1914                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1915                                 fieldtype = fi->type;
1916                                 disp      = fi->offset;
1917                         }
1918
1919                         /* implicit null-pointer check */
1920                         switch (fieldtype) {
1921                         case TYPE_INT:
1922                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1923                                 M_ILD32(d, s1, disp);
1924                                 break;
1925                         case TYPE_LNG:
1926                         case TYPE_ADR:
1927                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1928                                 M_LLD32(d, s1, disp);
1929                                 break;
1930                         case TYPE_FLT:
1931                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1932                                 M_FLD32(d, s1, disp);
1933                                 break;
1934                         case TYPE_DBL:
1935                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1936                                 M_DLD32(d, s1, disp);
1937                                 break;
1938                         }
1939                         emit_store_dst(jd, iptr, d);
1940                         break;
1941
1942                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1943
1944                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1945                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1946
1947                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1948                                 uf        = iptr->sx.s23.s3.uf;
1949                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1950                                 disp      = 0;
1951
1952 /*                              PROFILE_CYCLE_STOP; */
1953
1954                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1955
1956 /*                              PROFILE_CYCLE_START; */
1957                         }
1958                         else {
1959                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1960                                 fieldtype = fi->type;
1961                                 disp      = fi->offset;
1962                         }
1963
1964                         /* implicit null-pointer check */
1965                         switch (fieldtype) {
1966                         case TYPE_INT:
1967                                 M_IST32(s2, s1, disp);
1968                                 break;
1969                         case TYPE_LNG:
1970                         case TYPE_ADR:
1971                                 M_LST32(s2, s1, disp);
1972                                 break;
1973                         case TYPE_FLT:
1974                                 M_FST32(s2, s1, disp);
1975                                 break;
1976                         case TYPE_DBL:
1977                                 M_DST32(s2, s1, disp);
1978                                 break;
1979                         }
1980                         break;
1981
1982                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
1983                                           /* val = value (in current instruction)     */
1984                                           /* following NOP)                           */
1985
1986                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1987
1988                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1989                                 uf        = iptr->sx.s23.s3.uf;
1990                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1991                                 disp      = 0;
1992
1993 /*                              PROFILE_CYCLE_STOP; */
1994
1995                                 patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
1996
1997 /*                              PROFILE_CYCLE_START; */
1998                         }
1999                         else {
2000                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2001                                 fieldtype = fi->type;
2002                                 disp      = fi->offset;
2003                         }
2004
2005                         /* implicit null-pointer check */
2006                         switch (fieldtype) {
2007                         case TYPE_INT:
2008                         case TYPE_FLT:
2009                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2010                                 break;
2011                         case TYPE_LNG:
2012                         case TYPE_ADR:
2013                         case TYPE_DBL:
2014                                 /* XXX why no check for IS_IMM32? */
2015                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2016                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2017                                 break;
2018                         }
2019                         break;
2020
2021
2022                 /* branch operations **************************************************/
2023
2024                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2025
2026                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2027                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2028
2029                         PROFILE_CYCLE_STOP;
2030
2031 #ifdef ENABLE_VERIFIER
2032                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2033                                 uc = iptr->sx.s23.s2.uc;
2034
2035                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2036                         }
2037 #endif /* ENABLE_VERIFIER */
2038
2039                         M_CALL_IMM(0);                            /* passing exception pc */
2040                         M_POP(REG_ITMP2_XPC);
2041
2042                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2043                         M_JMP(REG_ITMP3);
2044                         break;
2045
2046                 case ICMD_GOTO:         /* ... ==> ...                                */
2047                 case ICMD_RET:
2048
2049                         emit_br(cd, iptr->dst.block);
2050                         ALIGNCODENOP;
2051                         break;
2052
2053                 case ICMD_JSR:          /* ... ==> ...                                */
2054
2055                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2056                         ALIGNCODENOP;
2057                         break;
2058
2059                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2060                 case ICMD_IFNONNULL:
2061
2062                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2063                         M_TEST(s1);
2064                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2065                         break;
2066
2067                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2068                 case ICMD_IFLT:
2069                 case ICMD_IFLE:
2070                 case ICMD_IFNE:
2071                 case ICMD_IFGT:
2072                 case ICMD_IFGE:
2073
2074                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2075                         M_ICMP_IMM(iptr->sx.val.i, s1);
2076                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2077                         break;
2078
2079                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2080                 case ICMD_IF_LNE:
2081                 case ICMD_IF_LLT:
2082                 case ICMD_IF_LGE:
2083                 case ICMD_IF_LGT:
2084                 case ICMD_IF_LLE:
2085
2086                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2087                         if (IS_IMM32(iptr->sx.val.l))
2088                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2089                         else {
2090                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2091                                 M_LCMP(REG_ITMP2, s1);
2092                         }
2093                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2094                         break;
2095
2096                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2097                 case ICMD_IF_ICMPNE:
2098                 case ICMD_IF_ICMPLT:
2099                 case ICMD_IF_ICMPGE:
2100                 case ICMD_IF_ICMPGT:
2101                 case ICMD_IF_ICMPLE:
2102
2103                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2104                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2105                         M_ICMP(s2, s1);
2106                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2107                         break;
2108
2109                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2110                 case ICMD_IF_ACMPNE:
2111
2112                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2113                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2114                         M_LCMP(s2, s1);
2115                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2116                         break;
2117
2118                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2119                 case ICMD_IF_LCMPNE:
2120                 case ICMD_IF_LCMPLT:
2121                 case ICMD_IF_LCMPGE:
2122                 case ICMD_IF_LCMPGT:
2123                 case ICMD_IF_LCMPLE:
2124
2125                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2126                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2127                         M_LCMP(s2, s1);
2128                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2129                         break;
2130
2131                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2132                 case ICMD_LRETURN:
2133
2134                         REPLACEMENT_POINT_RETURN(cd, iptr);
2135                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2136                         M_INTMOVE(s1, REG_RESULT);
2137                         goto nowperformreturn;
2138
2139                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2140
2141                         REPLACEMENT_POINT_RETURN(cd, iptr);
2142                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2143                         M_INTMOVE(s1, REG_RESULT);
2144
2145 #ifdef ENABLE_VERIFIER
2146                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2147                                 uc = iptr->sx.s23.s2.uc;
2148
2149                                 PROFILE_CYCLE_STOP;
2150
2151                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2152
2153                                 PROFILE_CYCLE_START;
2154                         }
2155 #endif /* ENABLE_VERIFIER */
2156                         goto nowperformreturn;
2157
2158                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2159                 case ICMD_DRETURN:
2160
2161                         REPLACEMENT_POINT_RETURN(cd, iptr);
2162                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2163                         M_FLTMOVE(s1, REG_FRESULT);
2164                         goto nowperformreturn;
2165
2166                 case ICMD_RETURN:      /* ...  ==> ...                                */
2167
2168                         REPLACEMENT_POINT_RETURN(cd, iptr);
2169
2170 nowperformreturn:
2171                         {
2172                         s4 i, p;
2173
2174                         p = cd->stackframesize;
2175
2176 #if !defined(NDEBUG)
2177                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2178                                 emit_verbosecall_exit(jd);
2179 #endif /* !defined(NDEBUG) */
2180
2181 #if defined(ENABLE_THREADS)
2182                         if (checksync && code_is_synchronized(code)) {
2183                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2184
2185                                 /* we need to save the proper return value */
2186                                 switch (iptr->opc) {
2187                                 case ICMD_IRETURN:
2188                                 case ICMD_ARETURN:
2189                                 case ICMD_LRETURN:
2190                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2191                                         break;
2192                                 case ICMD_FRETURN:
2193                                 case ICMD_DRETURN:
2194                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2195                                         break;
2196                                 }
2197
2198                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2199                                 M_CALL(REG_ITMP1);
2200
2201                                 /* and now restore the proper return value */
2202                                 switch (iptr->opc) {
2203                                 case ICMD_IRETURN:
2204                                 case ICMD_ARETURN:
2205                                 case ICMD_LRETURN:
2206                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2207                                         break;
2208                                 case ICMD_FRETURN:
2209                                 case ICMD_DRETURN:
2210                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2211                                         break;
2212                                 }
2213                         }
2214 #endif
2215
2216                         /* restore saved registers */
2217
2218                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2219                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2220                         }
2221                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2222                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2223                         }
2224
2225                         /* deallocate stack */
2226
2227                         if (cd->stackframesize)
2228                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2229
2230                         /* generate method profiling code */
2231
2232                         PROFILE_CYCLE_STOP;
2233
2234                         M_RET;
2235                         }
2236                         break;
2237
2238
2239                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2240                         {
2241                                 s4 i, l;
2242                                 branch_target_t *table;
2243
2244                                 table = iptr->dst.table;
2245
2246                                 l = iptr->sx.s23.s2.tablelow;
2247                                 i = iptr->sx.s23.s3.tablehigh;
2248
2249                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2250                                 M_INTMOVE(s1, REG_ITMP1);
2251
2252                                 if (l != 0)
2253                                         M_ISUB_IMM(l, REG_ITMP1);
2254
2255                                 /* number of targets */
2256                                 i = i - l + 1;
2257
2258                 /* range check */
2259
2260                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2261                                 emit_bugt(cd, table[0].block);
2262
2263                                 /* build jump table top down and use address of lowest entry */
2264
2265                                 table += i;
2266
2267                                 while (--i >= 0) {
2268                                         dseg_add_target(cd, table->block);
2269                                         --table;
2270                                 }
2271
2272                                 /* length of dataseg after last dseg_add_target is used
2273                                    by load */
2274
2275                                 M_MOV_IMM(0, REG_ITMP2);
2276                                 dseg_adddata(cd);
2277                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2278                                 M_JMP(REG_ITMP1);
2279                         }
2280                         break;
2281
2282
2283                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2284                         {
2285                                 s4 i;
2286                                 lookup_target_t *lookup;
2287
2288                                 lookup = iptr->dst.lookup;
2289
2290                                 i = iptr->sx.s23.s2.lookupcount;
2291
2292                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2293                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2294
2295                                 while (--i >= 0) {
2296                                         M_ICMP_IMM(lookup->value, s1);
2297                                         emit_beq(cd, lookup->target.block);
2298                                         lookup++;
2299                                 }
2300
2301                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2302                                 ALIGNCODENOP;
2303                         }
2304                         break;
2305
2306
2307                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2308
2309                         REPLACEMENT_POINT_FORGC_BUILTIN(cd, iptr);
2310
2311                         bte = iptr->sx.s23.s3.bte;
2312                         md  = bte->md;
2313                         goto gen_method;
2314
2315                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2316
2317                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2318                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2319                 case ICMD_INVOKEINTERFACE:
2320
2321                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2322
2323                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2324                                 lm = NULL;
2325                                 um = iptr->sx.s23.s3.um;
2326                                 md = um->methodref->parseddesc.md;
2327                         }
2328                         else {
2329                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2330                                 um = NULL;
2331                                 md = lm->parseddesc;
2332                         }
2333
2334 gen_method:
2335                         s3 = md->paramcount;
2336
2337                         MCODECHECK((20 * s3) + 128);
2338
2339                         /* copy arguments to registers or stack location */
2340
2341                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2342                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2343                                 d   = md->params[s3].regoff;
2344
2345                                 /* already preallocated (ARGVAR)? */
2346
2347                                 if (var->flags & PREALLOC)
2348                                         continue;
2349
2350                                 if (IS_INT_LNG_TYPE(var->type)) {
2351                                         if (!md->params[s3].inmemory) {
2352                                                 s1 = emit_load(jd, iptr, var, d);
2353                                                 M_INTMOVE(s1, d);
2354                                         }
2355                                         else {
2356                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2357                                                 M_LST(s1, REG_SP, d);
2358                                         }
2359                                 }
2360                                 else {
2361                                         if (!md->params[s3].inmemory) {
2362                                                 s1 = emit_load(jd, iptr, var, d);
2363                                                 M_FLTMOVE(s1, d);
2364                                         }
2365                                         else {
2366                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2367
2368                                                 if (IS_2_WORD_TYPE(var->type))
2369                                                         M_DST(s1, REG_SP, d);
2370                                                 else
2371                                                         M_FST(s1, REG_SP, d);
2372                                         }
2373                                 }
2374                         }
2375
2376                         /* generate method profiling code */
2377
2378                         PROFILE_CYCLE_STOP;
2379
2380                         switch (iptr->opc) {
2381                         case ICMD_BUILTIN:
2382                                 if (bte->stub == NULL) {
2383                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2384                                 }
2385                                 else {
2386                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2387                                 }
2388                                 M_CALL(REG_ITMP1);
2389                                 break;
2390
2391                         case ICMD_INVOKESPECIAL:
2392                                 emit_nullpointer_check(cd, iptr, REG_A0);
2393                                 /* fall through */
2394
2395                         case ICMD_INVOKESTATIC:
2396                                 if (lm == NULL) {
2397                                         disp = dseg_add_unique_address(cd, um);
2398
2399                                         patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
2400                                                                                   um, disp);
2401                                 }
2402                                 else {
2403                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2404                                 }
2405
2406                                 M_ALD(REG_ITMP2, RIP, disp);
2407                                 M_CALL(REG_ITMP2);
2408                                 break;
2409
2410                         case ICMD_INVOKEVIRTUAL:
2411                                 if (lm == NULL) {
2412                                         patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
2413
2414                                         s1 = 0;
2415                                 }
2416                                 else {
2417                                         s1 = OFFSET(vftbl_t, table[0]) +
2418                                                 sizeof(methodptr) * lm->vftblindex;
2419                                 }
2420
2421                                 /* implicit null-pointer check */
2422                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2423                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2424                                 M_CALL(REG_ITMP3);
2425                                 break;
2426
2427                         case ICMD_INVOKEINTERFACE:
2428                                 if (lm == NULL) {
2429                                         patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
2430
2431                                         s1 = 0;
2432                                         s2 = 0;
2433                                 }
2434                                 else {
2435                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2436                                                 sizeof(methodptr) * lm->clazz->index;
2437
2438                                         s2 = sizeof(methodptr) * (lm - lm->clazz->methods);
2439                                 }
2440
2441                                 /* implicit null-pointer check */
2442                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2443                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2444                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2445                                 M_CALL(REG_ITMP3);
2446                                 break;
2447                         }
2448
2449                         /* generate method profiling code */
2450
2451                         PROFILE_CYCLE_START;
2452
2453                         /* store size of call code in replacement point */
2454
2455                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2456                         REPLACEMENT_POINT_FORGC_BUILTIN_RETURN(cd, iptr);
2457
2458                         /* store return value */
2459
2460                         switch (md->returntype.type) {
2461                         case TYPE_INT:
2462                         case TYPE_LNG:
2463                         case TYPE_ADR:
2464                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2465                                 M_INTMOVE(REG_RESULT, s1);
2466                                 emit_store_dst(jd, iptr, s1);
2467                                 break;
2468                         case TYPE_FLT:
2469                         case TYPE_DBL:
2470                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2471                                 M_FLTMOVE(REG_FRESULT, s1);
2472                                 emit_store_dst(jd, iptr, s1);
2473                                 break;
2474                         default:
2475                                 /* TYPE_VOID */
2476                                 break;
2477                         }
2478                         break;
2479
2480
2481                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2482
2483                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2484                                 /* object type cast-check */
2485
2486                                 classinfo *super;
2487                                 s4         superindex;
2488
2489                                 s4 looptarget;
2490
2491                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2492                                         super      = NULL;
2493                                         superindex = 0;
2494                                 }
2495                                 else {
2496                                         super      = iptr->sx.s23.s3.c.cls;
2497                                         superindex = super->index;
2498                                 }
2499
2500                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2501
2502                                 /* if class is not resolved, check which code to call */
2503
2504                                 if (super == NULL) {
2505                                         M_TEST(s1);
2506                                         emit_label_beq(cd, BRANCH_LABEL_1);
2507
2508                                         patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2509                                                                                   iptr->sx.s23.s3.c.ref, 0);
2510
2511                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2512                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2513                                         emit_label_beq(cd, BRANCH_LABEL_2);
2514                                 }
2515
2516                                 /* interface checkcast code */
2517
2518                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2519                                         if (super != NULL) {
2520                                                 M_TEST(s1);
2521                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2522                                         }
2523
2524                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2525
2526                                         if (super == NULL) {
2527                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
2528                                                                                           iptr->sx.s23.s3.c.ref,
2529                                                                                           0);
2530                                         }
2531
2532                                         M_ILD32(REG_ITMP3,
2533                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2534                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2535                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2536
2537                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2538                                                         OFFSET(vftbl_t, interfacetable[0]) -
2539                                                         superindex * sizeof(methodptr*));
2540                                         M_TEST(REG_ITMP3);
2541                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2542
2543                                         if (super == NULL)
2544                                                 emit_label_br(cd, BRANCH_LABEL_4);
2545                                         else
2546                                                 emit_label(cd, BRANCH_LABEL_3);
2547                                 }
2548
2549                                 /* class checkcast code */
2550
2551                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2552                                         if (super == NULL) {
2553                                                 emit_label(cd, BRANCH_LABEL_2);
2554
2555                                                 cr   = iptr->sx.s23.s3.c.ref;
2556                                                 disp = dseg_add_unique_address(cd, cr);
2557
2558                                                 patcher_add_patch_ref(jd,
2559                                                                                           PATCHER_resolve_classref_to_vftbl,
2560                                                                                           cr, disp);
2561                                         }
2562                                         else {
2563                                                 M_TEST(s1);
2564                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2565
2566                                                 disp = dseg_add_address(cd, super->vftbl);
2567                                         }
2568
2569                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2570                                         M_ALD(REG_ITMP3, RIP, disp);
2571
2572                                         if (super == NULL || super->vftbl->subtype_depth >= DISPLAY_SIZE) {
2573                                         M_AADD_IMM(-32, REG_SP); /* need some stack */
2574                                         if (s1 == REG_ITMP1)
2575                                                 M_AST(REG_ITMP1, REG_SP, -8 + 32);
2576
2577                                         M_ALD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_offset));
2578
2579                                         *(cd->mcodeptr++) = 0x4d;
2580                                         *(cd->mcodeptr++) = 0x3b;
2581                                         *(cd->mcodeptr++) = 0x1c;
2582                                         *(cd->mcodeptr++) = 0x02;
2583                                         /* cmp (ITMP2, ITMP1, 1), ITMP3 */
2584
2585                                         emit_label_beq(cd, BRANCH_LABEL_6); /* good */
2586
2587                                         M_LCMP_IMM(OFFSET(vftbl_t, subtype_display[DISPLAY_SIZE]), REG_ITMP1);
2588                                         emit_classcast_check(cd, iptr, BRANCH_NE, REG_ITMP3, s1);
2589
2590                                         M_AST(REG_ITMP3, REG_SP, -16 + 32);
2591                                         M_AST_IMM32(0, REG_SP, -24 + 32);
2592                                         M_ALD(REG_ITMP1, REG_ITMP2, OFFSET(vftbl_t, subtype_overflow));
2593                                         looptarget = cd->mcodeptr - cd->mcodebase;
2594
2595                                         M_ALD(REG_ITMP3, REG_SP, -24 + 32);
2596                                         M_ICMP_MEMBASE(REG_ITMP2, OFFSET(vftbl_t, subtype_overflow_length), REG_ITMP3);
2597                                         emit_label_bge(cd, BRANCH_LABEL_9); /* throw */
2598
2599                                         *(cd->mcodeptr++) = 0x4e;
2600                                         *(cd->mcodeptr++) = 0x8b;
2601                                         *(cd->mcodeptr++) = 0x1c;
2602                                         *(cd->mcodeptr++) = 0xd8;
2603                                         /* movq (ITMP1, ITMP3, 8), ITMP3 */
2604
2605                                         M_LCMP_MEMBASE(REG_SP, -16 + 32, REG_ITMP3);
2606                                         emit_label_beq(cd, BRANCH_LABEL_7); /* good, pop */
2607
2608                                         M_LINC_MEMBASE(REG_SP, -24 + 32);
2609                                         M_JMP_IMM2(looptarget - (cd->mcodeptr - cd->mcodebase) - 2); /* 1 byte displacement */
2610
2611                                         emit_label(cd, BRANCH_LABEL_9);
2612                                         M_AADD_IMM(32, REG_SP); /* restore stack frame */
2613                                         M_ALD_MEM(REG_ITMP3, TRAP_ClassCastException);
2614
2615                                         emit_label(cd, BRANCH_LABEL_7);
2616
2617                                         emit_label(cd, BRANCH_LABEL_6);
2618
2619                                         if (s1 == REG_ITMP1)
2620                                                 M_ALD(REG_ITMP1, REG_SP, -8 + 32);
2621                                         M_AADD_IMM(32, REG_SP);
2622                                         }
2623                                         else {
2624                                                 assert(super->vftbl->subtype_offset < 0x80);
2625                                                 *(cd->mcodeptr++) = 0x4d;
2626                                                 *(cd->mcodeptr++) = 0x3b;
2627                                                 *(cd->mcodeptr++) = 0x5a;
2628                                                 *(cd->mcodeptr++) = super->vftbl->subtype_offset;
2629                                                 /* cmp off(ITMP1), ITMP2 */
2630
2631                                                 emit_classcast_check(cd, iptr, BRANCH_NE, REG_ITMP3, s1);
2632                                         }
2633
2634                                         if (super != NULL)
2635                                                 emit_label(cd, BRANCH_LABEL_5);
2636                                 }
2637
2638                                 if (super == NULL) {
2639                                         emit_label(cd, BRANCH_LABEL_1);
2640                                         emit_label(cd, BRANCH_LABEL_4);
2641                                 }
2642
2643                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2644                         }
2645                         else {
2646                                 /* array type cast-check */
2647
2648                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2649                                 M_INTMOVE(s1, REG_A0);
2650
2651                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2652                                         cr   = iptr->sx.s23.s3.c.ref;
2653                                         disp = dseg_add_unique_address(cd, cr);
2654
2655                                         patcher_add_patch_ref(jd,
2656                                                                                   PATCHER_resolve_classref_to_classinfo,
2657                                                                                   cr, disp);
2658                                 }
2659                                 else {
2660                                         disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2661                                 }
2662
2663                                 M_ALD(REG_A1, RIP, disp);
2664                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2665                                 M_CALL(REG_ITMP1);
2666
2667                                 /* s1 may have been destroyed over the function call */
2668                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2669                                 M_TEST(REG_RESULT);
2670                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2671
2672                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2673                         }
2674
2675                         M_INTMOVE(s1, d);
2676                         emit_store_dst(jd, iptr, d);
2677                         break;
2678
2679                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2680
2681                         {
2682                         classinfo *super;
2683                         s4         superindex;
2684
2685                         s4 looptarget;
2686
2687                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2688                                 super      = NULL;
2689                                 superindex = 0;
2690                         }
2691                         else {
2692                                 super      = iptr->sx.s23.s3.c.cls;
2693                                 superindex = super->index;
2694                         }
2695
2696                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2697                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2698
2699                         if (s1 == d) {
2700                                 M_INTMOVE(s1, REG_ITMP1);
2701                                 s1 = REG_ITMP1;
2702                         }
2703
2704                         M_CLR(d);
2705
2706                         /* if class is not resolved, check which code to call */
2707
2708                         if (super == NULL) {
2709                                 M_TEST(s1);
2710                                 emit_label_beq(cd, BRANCH_LABEL_1);
2711
2712                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2713                                                                           iptr->sx.s23.s3.c.ref, 0);
2714
2715                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2716                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2717                                 emit_label_beq(cd, BRANCH_LABEL_2);
2718                         }
2719
2720                         /* interface instanceof code */
2721
2722                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2723                                 if (super != NULL) {
2724                                         M_TEST(s1);
2725                                         emit_label_beq(cd, BRANCH_LABEL_3);
2726                                 }
2727
2728                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2729
2730                                 if (super == NULL) {
2731                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
2732                                                                                   iptr->sx.s23.s3.c.ref, 0);
2733                                 }
2734
2735                                 M_ILD32(REG_ITMP3,
2736                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2737                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2738
2739                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2740
2741                                 M_BLE(a);
2742                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2743                                                 OFFSET(vftbl_t, interfacetable[0]) -
2744                                                 superindex * sizeof(methodptr*));
2745                                 M_TEST(REG_ITMP1);
2746                                 M_SETNE(d);
2747
2748                                 if (super == NULL)
2749                                         emit_label_br(cd, BRANCH_LABEL_4);
2750                                 else
2751                                         emit_label(cd, BRANCH_LABEL_3);
2752                         }
2753
2754                         /* class instanceof code */
2755
2756                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2757                                 if (super == NULL) {
2758                                         emit_label(cd, BRANCH_LABEL_2);
2759
2760                                         cr   = iptr->sx.s23.s3.c.ref;
2761                                         disp = dseg_add_unique_address(cd, cr);
2762
2763                                         patcher_add_patch_ref(jd,
2764                                                                                   PATCHER_resolve_classref_to_vftbl,
2765                                                                                   cr, disp);
2766                                 }
2767                                 else {
2768                                         M_TEST(s1);
2769                                         emit_label_beq(cd, BRANCH_LABEL_5);
2770
2771                                         disp = dseg_add_address(cd, super->vftbl);
2772                                 }
2773
2774                                 M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2775                                 M_ALD(REG_ITMP3, RIP, disp);
2776
2777                                 if (super == NULL || super->vftbl->subtype_depth >= DISPLAY_SIZE) {
2778                                 M_AADD_IMM(-32, REG_SP); /* need some stack */
2779                                 M_ALD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_offset));
2780
2781                                 *(cd->mcodeptr++) = 0x4d;
2782                                 *(cd->mcodeptr++) = 0x3b;
2783                                 *(cd->mcodeptr++) = 0x1c;
2784                                 *(cd->mcodeptr++) = 0x02;
2785                                 /* cmp (ITMP2, ITMP1, 1), ITMP3 */
2786
2787                                 emit_label_bne(cd, BRANCH_LABEL_6);
2788                                 if (d == REG_ITMP2) {
2789                                         M_SETE(d);
2790                                         M_BSEXT(d, d);
2791                                 } else
2792                                         M_LINC(d);
2793                                 emit_label_br(cd, BRANCH_LABEL_7); /* ende true */
2794
2795                                 emit_label(cd, BRANCH_LABEL_6);
2796
2797                                 M_LCMP_IMM(OFFSET(vftbl_t, subtype_display[DISPLAY_SIZE]), REG_ITMP1);
2798                                 emit_label_bne(cd, BRANCH_LABEL_6); /* ende false */
2799
2800                                 M_AST(REG_ITMP3, REG_SP, -16 + 32);
2801                                 M_AST_IMM32(0, REG_SP, -24 + 32);
2802                                 M_ALD(REG_ITMP1, REG_ITMP2, OFFSET(vftbl_t, subtype_overflow));
2803                                 looptarget = cd->mcodeptr - cd->mcodebase;
2804
2805                                 M_ALD(REG_ITMP3, REG_SP, -24 + 32);
2806                                 M_ICMP_MEMBASE(REG_ITMP2, OFFSET(vftbl_t, subtype_overflow_length), REG_ITMP3);
2807                                 emit_label_bge(cd, BRANCH_LABEL_8); /* ende false */
2808
2809                                 *(cd->mcodeptr++) = 0x4e;
2810                                 *(cd->mcodeptr++) = 0x8b;
2811                                 *(cd->mcodeptr++) = 0x1c;
2812                                 *(cd->mcodeptr++) = 0xd8;
2813                                 /* movq (ITMP1, ITMP3, 8), ITMP3 */
2814
2815                                 M_LCMP_MEMBASE(REG_SP, -16 + 32, REG_ITMP3);
2816                                 emit_label_bne(cd, BRANCH_LABEL_9);
2817                                 if (d == REG_ITMP2) {
2818                                         M_SETE(d);
2819                                         M_BSEXT(d, d);
2820                                 } else
2821                                         M_LINC(d);
2822                                 emit_label_br(cd, BRANCH_LABEL_10); /* ende true */
2823                                 emit_label(cd, BRANCH_LABEL_9);
2824
2825                                 M_LINC_MEMBASE(REG_SP, -24 + 32);
2826                                 M_JMP_IMM2(looptarget - (cd->mcodeptr - cd->mcodebase) - 2); /* 1 byte displacement */
2827
2828                                 emit_label(cd, BRANCH_LABEL_8);
2829                                 emit_label(cd, BRANCH_LABEL_6);
2830
2831                                 if (d == REG_ITMP2)
2832                                         M_CLR(d);
2833
2834                                 emit_label(cd, BRANCH_LABEL_10);
2835                                 emit_label(cd, BRANCH_LABEL_7);
2836                                 M_AADD_IMM(32, REG_SP);
2837                                 }
2838                                 else {
2839                                         assert(super->vftbl->subtype_offset < 0x80);
2840                                         *(cd->mcodeptr++) = 0x4d;
2841                                         *(cd->mcodeptr++) = 0x3b;
2842                                         *(cd->mcodeptr++) = 0x5a;
2843                                         *(cd->mcodeptr++) = super->vftbl->subtype_offset;
2844                                         /* cmp off(ITMP1), ITMP2 */
2845
2846                                         M_SETE(d);
2847                                         if (d == REG_ITMP2)
2848                                                 M_BSEXT(d, d);
2849                                 }
2850
2851                                 if (super != NULL)
2852                                         emit_label(cd, BRANCH_LABEL_5);
2853                         }
2854
2855                         if (super == NULL) {
2856                                 emit_label(cd, BRANCH_LABEL_1);
2857                                 emit_label(cd, BRANCH_LABEL_4);
2858                         }
2859
2860                         emit_store_dst(jd, iptr, d);
2861                         }
2862                         break;
2863
2864                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2865
2866                         /* check for negative sizes and copy sizes to stack if necessary  */
2867
2868                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2869
2870                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2871
2872                                 /* copy SAVEDVAR sizes to stack */
2873                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2874
2875                                 /* Already Preallocated? */
2876                                 if (!(var->flags & PREALLOC)) {
2877                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2878                                         M_LST(s2, REG_SP, s1 * 8);
2879                                 }
2880                         }
2881
2882                         /* a0 = dimension count */
2883
2884                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2885
2886                         /* is a patcher function set? */
2887
2888                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2889                                 cr   = iptr->sx.s23.s3.c.ref;
2890                                 disp = dseg_add_unique_address(cd, cr);
2891
2892                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
2893                                                                           cr, disp);
2894                         }
2895                         else {
2896                                 disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2897                         }
2898
2899                         /* a1 = classinfo */
2900
2901                         M_ALD(REG_A1, RIP, disp);
2902
2903                         /* a2 = pointer to dimensions = stack pointer */
2904
2905                         M_MOV(REG_SP, REG_A2);
2906
2907                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2908                         M_CALL(REG_ITMP1);
2909
2910                         /* check for exception before result assignment */
2911
2912                         emit_exception_check(cd, iptr);
2913
2914                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2915                         M_INTMOVE(REG_RESULT, s1);
2916                         emit_store_dst(jd, iptr, s1);
2917                         break;
2918
2919                 default:
2920                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2921                                                                                    iptr->opc);
2922                         return false;
2923         } /* switch */
2924
2925         } /* for instruction */
2926
2927         MCODECHECK(512); /* XXX require a lower number? */
2928
2929         /* At the end of a basic block we may have to append some nops,
2930            because the patcher stub calling code might be longer than the
2931            actual instruction. So codepatching does not change the
2932            following block unintentionally. */
2933
2934         if (cd->mcodeptr < cd->lastmcodeptr) {
2935                 while (cd->mcodeptr < cd->lastmcodeptr) {
2936                         M_NOP;
2937                 }
2938         }
2939
2940         } /* if (bptr -> flags >= BBREACHED) */
2941         } /* for basic block */
2942
2943         /* Generate patcher traps. */
2944
2945         emit_patcher_traps(jd);
2946
2947         /* everything's ok */
2948
2949         return true;
2950 }
2951
2952
2953 /* codegen_emit_stub_native ****************************************************
2954
2955    Emits a stub routine which calls a native method.
2956
2957 *******************************************************************************/
2958
2959 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f, int skipparams)
2960 {
2961         methodinfo  *m;
2962         codeinfo    *code;
2963         codegendata *cd;
2964         methoddesc  *md;
2965         int          i, j;
2966         int          s1, s2;
2967         int          disp;
2968
2969         /* Sanity check. */
2970
2971         assert(f != NULL);
2972
2973         /* Get required compiler data. */
2974
2975         m    = jd->m;
2976         code = jd->code;
2977         cd   = jd->cd;
2978
2979         /* initialize variables */
2980
2981         md = m->parseddesc;
2982
2983         /* calculate stack frame size */
2984
2985         cd->stackframesize =
2986                 sizeof(stackframeinfo_t) / SIZEOF_VOID_P +
2987                 sizeof(localref_table) / SIZEOF_VOID_P +
2988                 md->paramcount +
2989                 (md->returntype.type == TYPE_VOID ? 0 : 1) +
2990                 nmd->memuse;
2991
2992         ALIGN_ODD(cd->stackframesize);              /* keep stack 16-byte aligned */
2993
2994         /* create method header */
2995
2996         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2997         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2998         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2999         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
3000         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
3001
3002 #if defined(ENABLE_PROFILING)
3003         /* generate native method profiling code */
3004
3005         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3006                 /* count frequency */
3007
3008                 M_MOV_IMM(code, REG_ITMP3);
3009                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3010         }
3011 #endif
3012
3013         /* generate stub code */
3014
3015         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3016
3017 #if defined(ENABLE_GC_CACAO)
3018         /* Save callee saved integer registers in stackframeinfo (GC may
3019            need to recover them during a collection). */
3020
3021         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
3022                 OFFSET(stackframeinfo_t, intregs);
3023
3024         for (i = 0; i < INT_SAV_CNT; i++)
3025                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3026 #endif
3027
3028         /* save integer and float argument registers */
3029
3030         for (i = 0; i < md->paramcount; i++) {
3031                 if (!md->params[i].inmemory) {
3032                         s1 = md->params[i].regoff;
3033
3034                         switch (md->paramtypes[i].type) {
3035                         case TYPE_INT:
3036                         case TYPE_LNG:
3037                         case TYPE_ADR:
3038                                 M_LST(s1, REG_SP, i * 8);
3039                                 break;
3040                         case TYPE_FLT:
3041                         case TYPE_DBL:
3042                                 M_DST(s1, REG_SP, i * 8);
3043                                 break;
3044                         }
3045                 }
3046         }
3047
3048         /* create dynamic stack info */
3049
3050         M_MOV(REG_SP, REG_A0);
3051         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3052         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3053         M_CALL(REG_ITMP1);
3054
3055         /* remember class argument */
3056
3057         if (m->flags & ACC_STATIC)
3058                 M_MOV(REG_RESULT, REG_ITMP2);
3059
3060         /* restore integer and float argument registers */
3061
3062         for (i = 0; i < md->paramcount; i++) {
3063                 if (!md->params[i].inmemory) {
3064                         s1 = md->params[i].regoff;
3065
3066                         switch (md->paramtypes[i].type) {
3067                         case TYPE_INT:
3068                         case TYPE_LNG:
3069                         case TYPE_ADR:
3070                                 M_LLD(s1, REG_SP, i * 8);
3071                                 break;
3072                         case TYPE_FLT:
3073                         case TYPE_DBL:
3074                                 M_DLD(s1, REG_SP, i * 8);
3075                                 break;
3076                         }
3077                 }
3078         }
3079
3080         /* Copy or spill arguments to new locations. */
3081
3082         for (i = md->paramcount - 1, j = i + skipparams; i >= 0; i--, j--) {
3083                 s2 = nmd->params[j].regoff;
3084
3085                 switch (md->paramtypes[i].type) {
3086                 case TYPE_INT:
3087                 case TYPE_LNG:
3088                 case TYPE_ADR:
3089                         if (!md->params[i].inmemory) {
3090                                 s1 = md->params[i].regoff;
3091
3092                                 if (!nmd->params[j].inmemory)
3093                                         M_INTMOVE(s1, s2);
3094                                 else
3095                                         M_LST(s1, REG_SP, s2);
3096                         }
3097                         else {
3098                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3099                                 M_LLD(REG_ITMP1, REG_SP, s1);
3100                                 M_LST(REG_ITMP1, REG_SP, s2);
3101                         }
3102                         break;
3103                 case TYPE_FLT:
3104                         /* We only copy spilled float arguments, as the float
3105                            argument registers keep unchanged. */
3106
3107                         if (md->params[i].inmemory) {
3108                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3109
3110                                 M_FLD(REG_FTMP1, REG_SP, s1);
3111                                 M_FST(REG_FTMP1, REG_SP, s2);
3112                         }
3113                         break;
3114                 case TYPE_DBL:
3115                         if (md->params[i].inmemory) {
3116                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3117                                 M_DLD(REG_FTMP1, REG_SP, s1);
3118                                 M_DST(REG_FTMP1, REG_SP, s2);
3119                         }
3120                         break;
3121                 }
3122         }
3123
3124         /* Handle native Java methods. */
3125
3126         if (m->flags & ACC_NATIVE) {
3127                 /* put class into second argument register */
3128
3129                 if (m->flags & ACC_STATIC)
3130                         M_MOV(REG_ITMP2, REG_A1);
3131
3132                 /* put env into first argument register */
3133
3134                 M_MOV_IMM(_Jv_env, REG_A0);
3135         }
3136
3137         /* Call the native function. */
3138
3139         disp = dseg_add_functionptr(cd, f);
3140         M_ALD(REG_ITMP1, RIP, disp);
3141         M_CALL(REG_ITMP1);
3142
3143         /* save return value */
3144
3145         switch (md->returntype.type) {
3146         case TYPE_INT:
3147         case TYPE_LNG:
3148         case TYPE_ADR:
3149                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3150                 break;
3151         case TYPE_FLT:
3152         case TYPE_DBL:
3153                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3154                 break;
3155         case TYPE_VOID:
3156                 break;
3157         }
3158
3159         /* remove native stackframe info */
3160
3161         M_MOV(REG_SP, REG_A0);
3162         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3163         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3164         M_CALL(REG_ITMP1);
3165         M_MOV(REG_RESULT, REG_ITMP3);
3166
3167         /* restore return value */
3168
3169         switch (md->returntype.type) {
3170         case TYPE_INT:
3171         case TYPE_LNG:
3172         case TYPE_ADR:
3173                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3174                 break;
3175         case TYPE_FLT:
3176         case TYPE_DBL:
3177                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3178                 break;
3179         case TYPE_VOID:
3180                 break;
3181         }
3182
3183 #if defined(ENABLE_GC_CACAO)
3184         /* Restore callee saved integer registers from stackframeinfo (GC
3185            might have modified them during a collection). */
3186
3187         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
3188                 OFFSET(stackframeinfo_t, intregs);
3189
3190         for (i = 0; i < INT_SAV_CNT; i++)
3191                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3192 #endif
3193
3194         /* remove stackframe */
3195
3196         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3197
3198         /* test for exception */
3199
3200         M_TEST(REG_ITMP3);
3201         M_BNE(1);
3202         M_RET;
3203
3204         /* handle exception */
3205
3206         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3207         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3208         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3209
3210         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3211         M_JMP(REG_ITMP3);
3212 }
3213
3214
3215 /*
3216  * These are local overrides for various environment variables in Emacs.
3217  * Please do not remove this and leave it at the end of the file, where
3218  * Emacs will automagically detect them.
3219  * ---------------------------------------------------------------------
3220  * Local variables:
3221  * mode: c
3222  * indent-tabs-mode: t
3223  * c-basic-offset: 4
3224  * tab-width: 4
3225  * End:
3226  * vim:noexpandtab:sw=4:ts=4:
3227  */