src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25 */
  26
  27
  28 #include "config.h"
  29
  30 #include <assert.h>
  31 #include <stdio.h>
  32 #include <stdint.h>
  33
  34 #include "vm/types.h"
  35
  36 #include "md-abi.h"
  37
  38 #include "vm/jit/x86_64/arch.h"
  39 #include "vm/jit/x86_64/codegen.h"
  40 #include "vm/jit/x86_64/emit.h"
  41
  42 #include "mm/memory.h"
  43
  44 #include "native/jni.h"
  45 #include "native/localref.h"
  46 #include "native/native.h"
  47
  48 #include "threads/lock-common.h"
  49
  50 #include "vm/builtin.h"
  51 #include "vm/exceptions.h"
  52 #include "vm/global.h"
  53 #include "vm/primitive.h"
  54 #include "vm/stringlocal.h"
  55 #include "vm/vm.h"
  56
  57 #include "vm/jit/abi.h"
  58 #include "vm/jit/asmpart.h"
  59 #include "vm/jit/code.h"
  60 #include "vm/jit/codegen-common.h"
  61 #include "vm/jit/dseg.h"
  62 #include "vm/jit/emit-common.h"
  63 #include "vm/jit/jit.h"
  64 #include "vm/jit/linenumbertable.h"
  65 #include "vm/jit/methodheader.h"
  66 #include "vm/jit/parse.h"
  67 #include "vm/jit/patcher-common.h"
  68 #include "vm/jit/reg.h"
  69 #include "vm/jit/replace.h"
  70 #include "vm/jit/stacktrace.h"
  71 #include "vm/jit/trap.h"
  72
  73 #if defined(ENABLE_LSRA)
  74 # include "vm/jit/allocator/lsra.h"
  75 #endif
  76
  77 #include "vmcore/loader.h"
  78 #include "vmcore/options.h"
  79 #include "vmcore/statistics.h"
  80
  81
  82 /* codegen_emit ****************************************************************
  83
  84    Generates machine code.
  85
  86 *******************************************************************************/
  87
  88 bool codegen_emit(jitdata *jd)
  89 {
  90         methodinfo         *m;
  91         codeinfo           *code;
  92         codegendata        *cd;
  93         registerdata       *rd;
  94         s4                  len, s1, s2, s3, d, disp;
  95         u2                  currentline;
  96         ptrint              a;
  97         varinfo            *var, *dst;
  98         basicblock         *bptr;
  99         instruction        *iptr;
 100         constant_classref  *cr;
 101         unresolved_class   *uc;
 102         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 103         unresolved_method  *um;
 104         builtintable_entry *bte;
 105         methoddesc         *md;
 106         fieldinfo          *fi;
 107         unresolved_field   *uf;
 108         s4                  fieldtype;
 109         s4                 varindex;
 110
 111         /* get required compiler data */
 112
 113         m    = jd->m;
 114         code = jd->code;
 115         cd   = jd->cd;
 116         rd   = jd->rd;
 117
 118         /* prevent compiler warnings */
 119
 120         d   = 0;
 121         lm  = NULL;
 122         um  = NULL;
 123         bte = NULL;
 124
 125         {
 126         s4 i, p, t, l;
 127         s4 savedregs_num;
 128
 129         savedregs_num = 0;
 130
 131         /* space to save used callee saved registers */
 132
 133         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 134         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 135
 136         cd->stackframesize = rd->memuse + savedregs_num;
 137
 138 #if defined(ENABLE_THREADS)
 139         /* space to save argument of monitor_enter */
 140
 141         if (checksync && code_is_synchronized(code))
 142                 cd->stackframesize++;
 143 #endif
 144
 145         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 146            native code e.g. libc or jni (alignment problems with
 147            movaps). */
 148
 149         if (!code_is_leafmethod(code) || opt_verbosecall)
 150                 cd->stackframesize |= 0x1;
 151
 152         /* create method header */
 153
 154         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 155         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 156
 157         code->synchronizedoffset = rd->memuse * 8;
 158
 159         if (code_is_leafmethod(code))
 160                 (void) dseg_add_unique_s4(cd, 1);                  /* IsLeaf          */
 161         else
 162                 (void) dseg_add_unique_s4(cd, 0);                  /* IsLeaf          */
 163
 164         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 165         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 166
 167 #if defined(ENABLE_PROFILING)
 168         /* generate method profiling code */
 169
 170         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 171                 /* count frequency */
 172
 173                 M_MOV_IMM(code, REG_ITMP3);
 174                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 175
 176                 PROFILE_CYCLE_START;
 177         }
 178 #endif
 179
 180         /* create stack frame (if necessary) */
 181
 182         if (cd->stackframesize)
 183                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 184
 185         /* save used callee saved registers */
 186
 187         p = cd->stackframesize;
 188         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 189                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 190         }
 191         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 192                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 193         }
 194
 195         /* take arguments out of register or stack frame */
 196
 197         md = m->parseddesc;
 198
 199         for (p = 0, l = 0; p < md->paramcount; p++) {
 200                 t = md->paramtypes[p].type;
 201
 202                 varindex = jd->local_map[l * 5 + t];
 203
 204                 l++;
 205                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 206                         l++;
 207
 208                 if (varindex == UNUSED)
 209                         continue;
 210
 211                 var = VAR(varindex);
 212
 213                 s1 = md->params[p].regoff;
 214
 215                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 216                         if (!md->params[p].inmemory) {           /* register arguments    */
 217                                 if (!IS_INMEMORY(var->flags))
 218                                         M_INTMOVE(s1, var->vv.regoff);
 219                                 else
 220                                     M_LST(s1, REG_SP, var->vv.regoff);
 221                         }
 222                         else {                                 /* stack arguments       */
 223                                 if (!IS_INMEMORY(var->flags))
 224                                         /* + 8 for return address */
 225                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 226                                 else
 227                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 228                         }
 229                 }
 230                 else {                                     /* floating args         */
 231                         if (!md->params[p].inmemory) {           /* register arguments    */
 232                                 if (!IS_INMEMORY(var->flags))
 233                                         M_FLTMOVE(s1, var->vv.regoff);
 234                                 else
 235                                         M_DST(s1, REG_SP, var->vv.regoff);
 236                         }
 237                         else {                                 /* stack arguments       */
 238                                 if (!IS_INMEMORY(var->flags))
 239                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 240                                 else
 241                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 242                         }
 243                 }
 244         }
 245
 246         /* save monitorenter argument */
 247
 248 #if defined(ENABLE_THREADS)
 249         if (checksync && code_is_synchronized(code)) {
 250                 /* stack offset for monitor argument */
 251
 252                 s1 = rd->memuse;
 253
 254                 if (opt_verbosecall) {
 255                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 256
 257                         for (p = 0; p < INT_ARG_CNT; p++)
 258                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 259
 260                         for (p = 0; p < FLT_ARG_CNT; p++)
 261                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 262
 263                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 264                 }
 265
 266                 /* decide which monitor enter function to call */
 267
 268                 if (m->flags & ACC_STATIC) {
 269                         M_MOV_IMM(&m->clazz->object.header, REG_A0);
 270                 }
 271                 else {
 272                         M_TEST(REG_A0);
 273                         M_BNE(8);
 274                         M_ALD_MEM(REG_A0, TRAP_NullPointerException);
 275                 }
 276
 277                 M_AST(REG_A0, REG_SP, s1 * 8);
 278                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 279                 M_CALL(REG_ITMP1);
 280
 281                 if (opt_verbosecall) {
 282                         for (p = 0; p < INT_ARG_CNT; p++)
 283                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 284
 285                         for (p = 0; p < FLT_ARG_CNT; p++)
 286                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 287
 288                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 289                 }
 290         }
 291 #endif
 292
 293 #if !defined(NDEBUG)
 294         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 295                 emit_verbosecall_enter(jd);
 296 #endif /* !defined(NDEBUG) */
 297
 298         }
 299
 300         /* end of header generation */
 301
 302         /* create replacement points */
 303
 304         REPLACEMENT_POINTS_INIT(cd, jd);
 305
 306         /* walk through all basic blocks */
 307
 308         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 309
 310                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 311
 312                 if (bptr->flags >= BBREACHED) {
 313
 314                 /* branch resolving */
 315
 316                 codegen_resolve_branchrefs(cd, bptr);
 317
 318                 /* handle replacement points */
 319
 320                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 321
 322                 /* copy interface registers to their destination */
 323
 324                 len = bptr->indepth;
 325                 MCODECHECK(512);
 326
 327 #if defined(ENABLE_PROFILING)
 328                 /* generate basicblock profiling code */
 329
 330                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 331                         /* count frequency */
 332
 333                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 334                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 335
 336                         /* if this is an exception handler, start profiling again */
 337
 338                         if (bptr->type == BBTYPE_EXH)
 339                                 PROFILE_CYCLE_START;
 340                 }
 341 #endif
 342
 343 #if defined(ENABLE_LSRA)
 344                 if (opt_lsra) {
 345                         while (len) {
 346                                 len--;
 347                                 src = bptr->invars[len];
 348                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 349                                         if (bptr->type == BBTYPE_EXH) {
 350 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 351                                                 if (!IS_INMEMORY(src->flags))
 352                                                         d= src->vv.regoff;
 353                                                 else
 354                                                         d=REG_ITMP1;
 355                                                 M_INTMOVE(REG_ITMP1, d);
 356                                                 emit_store(jd, NULL, src, d);
 357                                         }
 358                                 }
 359                         }
 360
 361                 } else {
 362 #endif
 363
 364                 while (len) {
 365                         len--;
 366                         var = VAR(bptr->invars[len]);
 367                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 368                                 if (bptr->type == BBTYPE_EXH) {
 369                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 370                                         M_INTMOVE(REG_ITMP1, d);
 371                                         emit_store(jd, NULL, var, d);
 372                                 }
 373                         }
 374                         else {
 375                                 assert((var->flags & INOUT));
 376                         }
 377                 }
 378 #if defined(ENABLE_LSRA)
 379                 }
 380 #endif
 381                 /* walk through all instructions */
 382
 383                 len = bptr->icount;
 384                 currentline = 0;
 385
 386                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 387                         if (iptr->line != currentline) {
 388                                 linenumbertable_list_entry_add(cd, iptr->line);
 389                                 currentline = iptr->line;
 390                         }
 391
 392                         MCODECHECK(1024);                         /* 1KB should be enough */
 393
 394                 switch (iptr->opc) {
 395                 case ICMD_NOP:        /* ...  ==> ...                                 */
 396                 case ICMD_POP:        /* ..., value  ==> ...                          */
 397                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 398                         break;
 399
 400                 case ICMD_INLINE_START:
 401
 402                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 403                         break;
 404
 405                 case ICMD_INLINE_BODY:
 406
 407                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 408                         linenumbertable_list_entry_add_inline_start(cd, iptr);
 409                         linenumbertable_list_entry_add(cd, iptr->line);
 410                         break;
 411
 412                 case ICMD_INLINE_END:
 413
 414                         linenumbertable_list_entry_add_inline_end(cd, iptr);
 415                         linenumbertable_list_entry_add(cd, iptr->line);
 416                         break;
 417
 418                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 419
 420                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 421                         emit_nullpointer_check(cd, iptr, s1);
 422                         break;
 423
 424                 /* constant operations ************************************************/
 425
 426                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 427
 428                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 429                         ICONST(d, iptr->sx.val.i);
 430                         emit_store_dst(jd, iptr, d);
 431                         break;
 432
 433                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 434
 435                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 436                         LCONST(d, iptr->sx.val.l);
 437                         emit_store_dst(jd, iptr, d);
 438                         break;
 439
 440                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 441
 442                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 443                         disp = dseg_add_float(cd, iptr->sx.val.f);
 444                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 445                         emit_store_dst(jd, iptr, d);
 446                         break;
 447
 448                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 449
 450                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 451                         disp = dseg_add_double(cd, iptr->sx.val.d);
 452                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 453                         emit_store_dst(jd, iptr, d);
 454                         break;
 455
 456                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 457
 458                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 459
 460                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 461                                 cr   = iptr->sx.val.c.ref;
 462                                 disp = dseg_add_unique_address(cd, cr);
 463
 464 /*                              PROFILE_CYCLE_STOP; */
 465
 466                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
 467                                                                           cr, disp);
 468
 469 /*                              PROFILE_CYCLE_START; */
 470
 471                                 M_ALD(d, RIP, disp);
 472                         }
 473                         else {
 474                                 if (iptr->sx.val.anyptr == 0) {
 475                                         M_CLR(d);
 476                                 }
 477                                 else {
 478                                         disp = dseg_add_address(cd, iptr->sx.val.anyptr);
 479                                         M_ALD(d, RIP, disp);
 480                                 }
 481                         }
 482                         emit_store_dst(jd, iptr, d);
 483                         break;
 484
 485
 486                 /* load/store/copy/move operations ************************************/
 487
 488                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 489                 case ICMD_ALOAD:      /* s1 = local variable                          */
 490                 case ICMD_LLOAD:
 491                 case ICMD_FLOAD:
 492                 case ICMD_DLOAD:
 493                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 494                 case ICMD_LSTORE:
 495                 case ICMD_FSTORE:
 496                 case ICMD_DSTORE:
 497                 case ICMD_COPY:
 498                 case ICMD_MOVE:
 499
 500                         emit_copy(jd, iptr);
 501                         break;
 502
 503                 case ICMD_ASTORE:
 504                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 505                                 emit_copy(jd, iptr);
 506                         break;
 507
 508                 /* integer operations *************************************************/
 509
 510                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 511
 512                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 513                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 514                         M_INTMOVE(s1, d);
 515                         M_INEG(d);
 516                         emit_store_dst(jd, iptr, d);
 517                         break;
 518
 519                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 520
 521                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 522                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 523                         M_INTMOVE(s1, d);
 524                         M_LNEG(d);
 525                         emit_store_dst(jd, iptr, d);
 526                         break;
 527
 528                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 529
 530                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 531                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 532                         M_ISEXT(s1, d);
 533                         emit_store_dst(jd, iptr, d);
 534                         break;
 535
 536                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 537
 538                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 539                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 540                         M_IMOV(s1, d);
 541                         emit_store_dst(jd, iptr, d);
 542                         break;
 543
 544                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 545
 546                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 547                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 548                         M_BSEXT(s1, d);
 549                         emit_store_dst(jd, iptr, d);
 550                         break;
 551
 552                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 553
 554                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 555                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 556                         M_CZEXT(s1, d);
 557                         emit_store_dst(jd, iptr, d);
 558                         break;
 559
 560                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 561
 562                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 563                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 564                         M_SSEXT(s1, d);
 565                         emit_store_dst(jd, iptr, d);
 566                         break;
 567
 568
 569                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 570
 571                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 572                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 573                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 574                         if (s2 == d)
 575                                 M_IADD(s1, d);
 576                         else {
 577                                 M_INTMOVE(s1, d);
 578                                 M_IADD(s2, d);
 579                         }
 580                         emit_store_dst(jd, iptr, d);
 581                         break;
 582
 583                 case ICMD_IINC:
 584                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 585                                       /* sx.val.i = constant                             */
 586
 587                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 588                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 589
 590                         /* Using inc and dec is not faster than add (tested with
 591                            sieve). */
 592
 593                         M_INTMOVE(s1, d);
 594                         M_IADD_IMM(iptr->sx.val.i, d);
 595                         emit_store_dst(jd, iptr, d);
 596                         break;
 597
 598                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 599
 600                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 601                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 602                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 603                         if (s2 == d)
 604                                 M_LADD(s1, d);
 605                         else {
 606                                 M_INTMOVE(s1, d);
 607                                 M_LADD(s2, d);
 608                         }
 609                         emit_store_dst(jd, iptr, d);
 610                         break;
 611
 612                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 613                                       /* sx.val.l = constant                             */
 614
 615                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 616                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 617                         M_INTMOVE(s1, d);
 618                         if (IS_IMM32(iptr->sx.val.l))
 619                                 M_LADD_IMM(iptr->sx.val.l, d);
 620                         else {
 621                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 622                                 M_LADD(REG_ITMP2, d);
 623                         }
 624                         emit_store_dst(jd, iptr, d);
 625                         break;
 626
 627                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 628
 629                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 630                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 631                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 632                         if (s2 == d) {
 633                                 M_INTMOVE(s1, REG_ITMP1);
 634                                 M_ISUB(s2, REG_ITMP1);
 635                                 M_INTMOVE(REG_ITMP1, d);
 636                         } else {
 637                                 M_INTMOVE(s1, d);
 638                                 M_ISUB(s2, d);
 639                         }
 640                         emit_store_dst(jd, iptr, d);
 641                         break;
 642
 643                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 644                                       /* sx.val.i = constant                             */
 645
 646                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 647                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 648                         M_INTMOVE(s1, d);
 649                         M_ISUB_IMM(iptr->sx.val.i, d);
 650                         emit_store_dst(jd, iptr, d);
 651                         break;
 652
 653                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 654
 655                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 656                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 657                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 658                         if (s2 == d) {
 659                                 M_INTMOVE(s1, REG_ITMP1);
 660                                 M_LSUB(s2, REG_ITMP1);
 661                                 M_INTMOVE(REG_ITMP1, d);
 662                         } else {
 663                                 M_INTMOVE(s1, d);
 664                                 M_LSUB(s2, d);
 665                         }
 666                         emit_store_dst(jd, iptr, d);
 667                         break;
 668
 669                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 670                                       /* sx.val.l = constant                             */
 671
 672                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 673                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 674                         M_INTMOVE(s1, d);
 675                         if (IS_IMM32(iptr->sx.val.l))
 676                                 M_LSUB_IMM(iptr->sx.val.l, d);
 677                         else {
 678                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 679                                 M_LSUB(REG_ITMP2, d);
 680                         }
 681                         emit_store_dst(jd, iptr, d);
 682                         break;
 683
 684                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 685
 686                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 687                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 688                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 689                         if (s2 == d)
 690                                 M_IMUL(s1, d);
 691                         else {
 692                                 M_INTMOVE(s1, d);
 693                                 M_IMUL(s2, d);
 694                         }
 695                         emit_store_dst(jd, iptr, d);
 696                         break;
 697
 698                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 699                                       /* sx.val.i = constant                             */
 700
 701                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 702                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 703                         if (iptr->sx.val.i == 2) {
 704                                 M_INTMOVE(s1, d);
 705                                 M_ISLL_IMM(1, d);
 706                         } else
 707                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 708                         emit_store_dst(jd, iptr, d);
 709                         break;
 710
 711                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 712
 713                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 714                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 715                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 716                         if (s2 == d)
 717                                 M_LMUL(s1, d);
 718                         else {
 719                                 M_INTMOVE(s1, d);
 720                                 M_LMUL(s2, d);
 721                         }
 722                         emit_store_dst(jd, iptr, d);
 723                         break;
 724
 725                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 726                                       /* sx.val.l = constant                             */
 727
 728                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 729                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 730                         if (IS_IMM32(iptr->sx.val.l))
 731                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 732                         else {
 733                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 734                                 M_INTMOVE(s1, d);
 735                                 M_LMUL(REG_ITMP2, d);
 736                         }
 737                         emit_store_dst(jd, iptr, d);
 738                         break;
 739
 740                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 741
 742                         s1 = emit_load_s1(jd, iptr, RAX);
 743                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 744                         d = codegen_reg_of_dst(jd, iptr, RAX);
 745
 746                         M_INTMOVE(s1, RAX);
 747                         M_INTMOVE(s2, REG_ITMP3);
 748                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 749
 750                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 751
 752                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 753                         M_BNE(4 + 6);
 754                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 755                         M_BEQ(1 + 3);                                          /* 6 bytes */
 756
 757                         emit_cltd(cd);                                         /* 1 byte  */
 758                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 759
 760                         M_INTMOVE(RAX, d);
 761                         emit_store_dst(jd, iptr, d);
 762                         dst = VAROP(iptr->dst);
 763                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 764                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 765                         break;
 766
 767                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 768
 769                         s1 = emit_load_s1(jd, iptr, RAX);
 770                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 771                         d = codegen_reg_of_dst(jd, iptr, RDX);
 772
 773                         M_INTMOVE(s1, RAX);
 774                         M_INTMOVE(s2, REG_ITMP3);
 775                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 776
 777                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 778
 779                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 780                         M_BNE(3 + 4 + 6);
 781                         M_CLR(RDX);                                            /* 3 bytes */
 782                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 783                         M_BEQ(1 + 3);                                          /* 6 bytes */
 784
 785                         emit_cltd(cd);                                         /* 1 byte  */
 786                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 787
 788                         M_INTMOVE(RDX, d);
 789                         emit_store_dst(jd, iptr, d);
 790                         dst = VAROP(iptr->dst);
 791                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 792                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 793                         break;
 794
 795                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 796                                       /* sx.val.i = constant                             */
 797
 798                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 799                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 800                         M_INTMOVE(s1, REG_ITMP1);
 801                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 802                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 803                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 804                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 805                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 806                         emit_store_dst(jd, iptr, d);
 807                         break;
 808
 809                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 810                                       /* sx.val.i = constant                             */
 811
 812                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 813                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 814                         M_INTMOVE(s1, REG_ITMP1);
 815                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 816                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 817                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 818                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 819                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 820                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 821                         emit_store_dst(jd, iptr, d);
 822                         break;
 823
 824
 825                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 826
 827                         s1 = emit_load_s1(jd, iptr, RAX);
 828                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 829                         d = codegen_reg_of_dst(jd, iptr, RAX);
 830
 831                         M_INTMOVE(s1, RAX);
 832                         M_INTMOVE(s2, REG_ITMP3);
 833                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 834
 835                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 836
 837                         /* check as described in jvm spec */
 838                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 839                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 840                         M_BNE(4 + 6);
 841                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 842                         M_BEQ(2 + 3);                                          /* 6 bytes */
 843
 844                         emit_cqto(cd);                                         /* 2 bytes */
 845                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 846
 847                         M_INTMOVE(RAX, d);
 848                         emit_store_dst(jd, iptr, d);
 849                         dst = VAROP(iptr->dst);
 850                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 851                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 852                         break;
 853
 854                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 855
 856                         s1 = emit_load_s1(jd, iptr, RAX);
 857                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 858                         d = codegen_reg_of_dst(jd, iptr, RDX);
 859
 860                         M_INTMOVE(s1, RAX);
 861                         M_INTMOVE(s2, REG_ITMP3);
 862                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 863
 864                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 865
 866                         /* check as described in jvm spec */
 867                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 868                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 869                         M_BNE(3 + 4 + 6);
 870                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 871                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 872                         M_BEQ(2 + 3);                                          /* 6 bytes */
 873
 874                         emit_cqto(cd);                                         /* 2 bytes */
 875                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 876
 877                         M_INTMOVE(RDX, d);
 878                         emit_store_dst(jd, iptr, d);
 879                         dst = VAROP(iptr->dst);
 880                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 881                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 882                         break;
 883
 884                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 885                                       /* sx.val.i = constant                             */
 886
 887                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 888                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 889                         M_INTMOVE(s1, REG_ITMP1);
 890                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 891                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 892                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 893                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 894                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 895                         emit_store_dst(jd, iptr, d);
 896                         break;
 897
 898                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 899                                       /* sx.val.l = constant                             */
 900
 901                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 902                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 903                         M_INTMOVE(s1, REG_ITMP1);
 904                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 905                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 906                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 907                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 908                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 909                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 910                         emit_store_dst(jd, iptr, d);
 911                         break;
 912
 913                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 914
 915                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 916                         emit_ishift(jd, SHIFT_SHL, iptr);
 917                         break;
 918
 919                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 920                                       /* sx.val.i = constant                             */
 921
 922                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 923                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 924                         M_INTMOVE(s1, d);
 925                         M_ISLL_IMM(iptr->sx.val.i, d);
 926                         emit_store_dst(jd, iptr, d);
 927                         break;
 928
 929                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 930
 931                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 932                         emit_ishift(jd, SHIFT_SAR, iptr);
 933                         break;
 934
 935                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 936                                       /* sx.val.i = constant                             */
 937
 938                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 939                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 940                         M_INTMOVE(s1, d);
 941                         M_ISRA_IMM(iptr->sx.val.i, d);
 942                         emit_store_dst(jd, iptr, d);
 943                         break;
 944
 945                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 946
 947                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 948                         emit_ishift(jd, SHIFT_SHR, iptr);
 949                         break;
 950
 951                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 952                                       /* sx.val.i = constant                             */
 953
 954                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 955                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 956                         M_INTMOVE(s1, d);
 957                         M_ISRL_IMM(iptr->sx.val.i, d);
 958                         emit_store_dst(jd, iptr, d);
 959                         break;
 960
 961                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 962
 963                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 964                         emit_lshift(jd, SHIFT_SHL, iptr);
 965                         break;
 966
 967         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 968                                           /* sx.val.i = constant                             */
 969
 970                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 971                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 972                         M_INTMOVE(s1, d);
 973                         M_LSLL_IMM(iptr->sx.val.i, d);
 974                         emit_store_dst(jd, iptr, d);
 975                         break;
 976
 977                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 978
 979                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 980                         emit_lshift(jd, SHIFT_SAR, iptr);
 981                         break;
 982
 983                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 984                                       /* sx.val.i = constant                             */
 985
 986                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 987                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 988                         M_INTMOVE(s1, d);
 989                         M_LSRA_IMM(iptr->sx.val.i, d);
 990                         emit_store_dst(jd, iptr, d);
 991                         break;
 992
 993                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 994
 995                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 996                         emit_lshift(jd, SHIFT_SHR, iptr);
 997                         break;
 998
 999                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1000                                       /* sx.val.l = constant                             */
1001
1002                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1003                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1004                         M_INTMOVE(s1, d);
1005                         M_LSRL_IMM(iptr->sx.val.i, d);
1006                         emit_store_dst(jd, iptr, d);
1007                         break;
1008
1009                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1010
1011                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1012                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1013                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1014                         if (s2 == d)
1015                                 M_IAND(s1, d);
1016                         else {
1017                                 M_INTMOVE(s1, d);
1018                                 M_IAND(s2, d);
1019                         }
1020                         emit_store_dst(jd, iptr, d);
1021                         break;
1022
1023                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1024                                       /* sx.val.i = constant                             */
1025
1026                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1027                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1028                         M_INTMOVE(s1, d);
1029                         M_IAND_IMM(iptr->sx.val.i, d);
1030                         emit_store_dst(jd, iptr, d);
1031                         break;
1032
1033                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1034
1035                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1036                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1037                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1038                         if (s2 == d)
1039                                 M_LAND(s1, d);
1040                         else {
1041                                 M_INTMOVE(s1, d);
1042                                 M_LAND(s2, d);
1043                         }
1044                         emit_store_dst(jd, iptr, d);
1045                         break;
1046
1047                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1048                                       /* sx.val.l = constant                             */
1049
1050                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1051                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1052                         M_INTMOVE(s1, d);
1053                         if (IS_IMM32(iptr->sx.val.l))
1054                                 M_LAND_IMM(iptr->sx.val.l, d);
1055                         else {
1056                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1057                                 M_LAND(REG_ITMP2, d);
1058                         }
1059                         emit_store_dst(jd, iptr, d);
1060                         break;
1061
1062                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1063
1064                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1065                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1066                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1067                         if (s2 == d)
1068                                 M_IOR(s1, d);
1069                         else {
1070                                 M_INTMOVE(s1, d);
1071                                 M_IOR(s2, d);
1072                         }
1073                         emit_store_dst(jd, iptr, d);
1074                         break;
1075
1076                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1077                                       /* sx.val.i = constant                             */
1078
1079                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1080                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1081                         M_INTMOVE(s1, d);
1082                         M_IOR_IMM(iptr->sx.val.i, d);
1083                         emit_store_dst(jd, iptr, d);
1084                         break;
1085
1086                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1087
1088                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1089                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1090                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1091                         if (s2 == d)
1092                                 M_LOR(s1, d);
1093                         else {
1094                                 M_INTMOVE(s1, d);
1095                                 M_LOR(s2, d);
1096                         }
1097                         emit_store_dst(jd, iptr, d);
1098                         break;
1099
1100                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1101                                       /* sx.val.l = constant                             */
1102
1103                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1104                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1105                         M_INTMOVE(s1, d);
1106                         if (IS_IMM32(iptr->sx.val.l))
1107                                 M_LOR_IMM(iptr->sx.val.l, d);
1108                         else {
1109                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1110                                 M_LOR(REG_ITMP2, d);
1111                         }
1112                         emit_store_dst(jd, iptr, d);
1113                         break;
1114
1115                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1116
1117                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1118                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1119                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1120                         if (s2 == d)
1121                                 M_IXOR(s1, d);
1122                         else {
1123                                 M_INTMOVE(s1, d);
1124                                 M_IXOR(s2, d);
1125                         }
1126                         emit_store_dst(jd, iptr, d);
1127                         break;
1128
1129                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1130                                       /* sx.val.i = constant                             */
1131
1132                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1133                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1134                         M_INTMOVE(s1, d);
1135                         M_IXOR_IMM(iptr->sx.val.i, d);
1136                         emit_store_dst(jd, iptr, d);
1137                         break;
1138
1139                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1140
1141                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1142                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1143                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1144                         if (s2 == d)
1145                                 M_LXOR(s1, d);
1146                         else {
1147                                 M_INTMOVE(s1, d);
1148                                 M_LXOR(s2, d);
1149                         }
1150                         emit_store_dst(jd, iptr, d);
1151                         break;
1152
1153                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1154                                       /* sx.val.l = constant                             */
1155
1156                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1157                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1158                         M_INTMOVE(s1, d);
1159                         if (IS_IMM32(iptr->sx.val.l))
1160                                 M_LXOR_IMM(iptr->sx.val.l, d);
1161                         else {
1162                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1163                                 M_LXOR(REG_ITMP2, d);
1164                         }
1165                         emit_store_dst(jd, iptr, d);
1166                         break;
1167
1168
1169                 /* floating operations ************************************************/
1170
1171                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1172
1173                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1174                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1175                         disp = dseg_add_s4(cd, 0x80000000);
1176                         M_FLTMOVE(s1, d);
1177                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1178                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1179                         emit_store_dst(jd, iptr, d);
1180                         break;
1181
1182                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1183
1184                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1185                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1186                         disp = dseg_add_s8(cd, 0x8000000000000000);
1187                         M_FLTMOVE(s1, d);
1188                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1189                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1190                         emit_store_dst(jd, iptr, d);
1191                         break;
1192
1193                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1194
1195                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1196                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1197                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1198                         if (s2 == d)
1199                                 M_FADD(s1, d);
1200                         else {
1201                                 M_FLTMOVE(s1, d);
1202                                 M_FADD(s2, d);
1203                         }
1204                         emit_store_dst(jd, iptr, d);
1205                         break;
1206
1207                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1208
1209                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1210                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1211                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1212                         if (s2 == d)
1213                                 M_DADD(s1, d);
1214                         else {
1215                                 M_FLTMOVE(s1, d);
1216                                 M_DADD(s2, d);
1217                         }
1218                         emit_store_dst(jd, iptr, d);
1219                         break;
1220
1221                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1222
1223                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1224                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1225                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1226                         if (s2 == d) {
1227                                 M_FLTMOVE(s2, REG_FTMP2);
1228                                 s2 = REG_FTMP2;
1229                         }
1230                         M_FLTMOVE(s1, d);
1231                         M_FSUB(s2, d);
1232                         emit_store_dst(jd, iptr, d);
1233                         break;
1234
1235                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1236
1237                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1238                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1239                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1240                         if (s2 == d) {
1241                                 M_FLTMOVE(s2, REG_FTMP2);
1242                                 s2 = REG_FTMP2;
1243                         }
1244                         M_FLTMOVE(s1, d);
1245                         M_DSUB(s2, d);
1246                         emit_store_dst(jd, iptr, d);
1247                         break;
1248
1249                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1250
1251                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1252                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1253                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1254                         if (s2 == d)
1255                                 M_FMUL(s1, d);
1256                         else {
1257                                 M_FLTMOVE(s1, d);
1258                                 M_FMUL(s2, d);
1259                         }
1260                         emit_store_dst(jd, iptr, d);
1261                         break;
1262
1263                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1264
1265                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1266                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1267                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1268                         if (s2 == d)
1269                                 M_DMUL(s1, d);
1270                         else {
1271                                 M_FLTMOVE(s1, d);
1272                                 M_DMUL(s2, d);
1273                         }
1274                         emit_store_dst(jd, iptr, d);
1275                         break;
1276
1277                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1278
1279                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1280                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1281                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1282                         if (s2 == d) {
1283                                 M_FLTMOVE(s2, REG_FTMP2);
1284                                 s2 = REG_FTMP2;
1285                         }
1286                         M_FLTMOVE(s1, d);
1287                         M_FDIV(s2, d);
1288                         emit_store_dst(jd, iptr, d);
1289                         break;
1290
1291                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1292
1293                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1294                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1295                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1296                         if (s2 == d) {
1297                                 M_FLTMOVE(s2, REG_FTMP2);
1298                                 s2 = REG_FTMP2;
1299                         }
1300                         M_FLTMOVE(s1, d);
1301                         M_DDIV(s2, d);
1302                         emit_store_dst(jd, iptr, d);
1303                         break;
1304
1305                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1306
1307                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1308                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1309                         M_CVTIF(s1, d);
1310                         emit_store_dst(jd, iptr, d);
1311                         break;
1312
1313                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1314
1315                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1316                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1317                         M_CVTID(s1, d);
1318                         emit_store_dst(jd, iptr, d);
1319                         break;
1320
1321                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1322
1323                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1324                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1325                         M_CVTLF(s1, d);
1326                         emit_store_dst(jd, iptr, d);
1327                         break;
1328
1329                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1330
1331                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1332                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1333                         M_CVTLD(s1, d);
1334                         emit_store_dst(jd, iptr, d);
1335                         break;
1336
1337                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1338
1339                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1340                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1341                         M_CVTFI(s1, d);
1342                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1343                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1344                                 ((REG_RESULT == d) ? 0 : 3);
1345                         M_BNE(disp);
1346                         M_FLTMOVE(s1, REG_FTMP1);
1347                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1348                         M_CALL(REG_ITMP2);
1349                         M_INTMOVE(REG_RESULT, d);
1350                         emit_store_dst(jd, iptr, d);
1351                         break;
1352
1353                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1354
1355                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1356                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1357                         M_CVTDI(s1, d);
1358                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1359                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1360                                 ((REG_RESULT == d) ? 0 : 3);
1361                         M_BNE(disp);
1362                         M_FLTMOVE(s1, REG_FTMP1);
1363                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1364                         M_CALL(REG_ITMP2);
1365                         M_INTMOVE(REG_RESULT, d);
1366                         emit_store_dst(jd, iptr, d);
1367                         break;
1368
1369                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1370
1371                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1372                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1373                         M_CVTFL(s1, d);
1374                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1375                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1376                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1377                                 ((REG_RESULT == d) ? 0 : 3);
1378                         M_BNE(disp);
1379                         M_FLTMOVE(s1, REG_FTMP1);
1380                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1381                         M_CALL(REG_ITMP2);
1382                         M_INTMOVE(REG_RESULT, d);
1383                         emit_store_dst(jd, iptr, d);
1384                         break;
1385
1386                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1387
1388                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1389                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1390                         M_CVTDL(s1, d);
1391                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1392                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1393                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1394                                 ((REG_RESULT == d) ? 0 : 3);
1395                         M_BNE(disp);
1396                         M_FLTMOVE(s1, REG_FTMP1);
1397                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1398                         M_CALL(REG_ITMP2);
1399                         M_INTMOVE(REG_RESULT, d);
1400                         emit_store_dst(jd, iptr, d);
1401                         break;
1402
1403                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1404
1405                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1406                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1407                         M_CVTFD(s1, d);
1408                         emit_store_dst(jd, iptr, d);
1409                         break;
1410
1411                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1412
1413                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1414                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1415                         M_CVTDF(s1, d);
1416                         emit_store_dst(jd, iptr, d);
1417                         break;
1418
1419                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1420                                           /* == => 0, < => 1, > => -1 */
1421
1422                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1423                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1424                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1425                         M_CLR(d);
1426                         M_MOV_IMM(1, REG_ITMP1);
1427                         M_MOV_IMM(-1, REG_ITMP2);
1428                         emit_ucomiss_reg_reg(cd, s1, s2);
1429                         M_CMOVULT(REG_ITMP1, d);
1430                         M_CMOVUGT(REG_ITMP2, d);
1431                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1432                         emit_store_dst(jd, iptr, d);
1433                         break;
1434
1435                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1436                                           /* == => 0, < => 1, > => -1 */
1437
1438                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1439                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1440                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1441                         M_CLR(d);
1442                         M_MOV_IMM(1, REG_ITMP1);
1443                         M_MOV_IMM(-1, REG_ITMP2);
1444                         emit_ucomiss_reg_reg(cd, s1, s2);
1445                         M_CMOVULT(REG_ITMP1, d);
1446                         M_CMOVUGT(REG_ITMP2, d);
1447                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1448                         emit_store_dst(jd, iptr, d);
1449                         break;
1450
1451                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1452                                           /* == => 0, < => 1, > => -1 */
1453
1454                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1455                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1456                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1457                         M_CLR(d);
1458                         M_MOV_IMM(1, REG_ITMP1);
1459                         M_MOV_IMM(-1, REG_ITMP2);
1460                         emit_ucomisd_reg_reg(cd, s1, s2);
1461                         M_CMOVULT(REG_ITMP1, d);
1462                         M_CMOVUGT(REG_ITMP2, d);
1463                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1464                         emit_store_dst(jd, iptr, d);
1465                         break;
1466
1467                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1468                                           /* == => 0, < => 1, > => -1 */
1469
1470                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1471                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1472                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1473                         M_CLR(d);
1474                         M_MOV_IMM(1, REG_ITMP1);
1475                         M_MOV_IMM(-1, REG_ITMP2);
1476                         emit_ucomisd_reg_reg(cd, s1, s2);
1477                         M_CMOVULT(REG_ITMP1, d);
1478                         M_CMOVUGT(REG_ITMP2, d);
1479                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1480                         emit_store_dst(jd, iptr, d);
1481                         break;
1482
1483
1484                 /* memory operations **************************************************/
1485
1486                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1487
1488                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1489                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1490                         /* implicit null-pointer check */
1491                         M_ILD(d, s1, OFFSET(java_array_t, size));
1492                         emit_store_dst(jd, iptr, d);
1493                         break;
1494
1495                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1496
1497                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1498                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1499                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1500                         /* implicit null-pointer check */
1501                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1502                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1503                         emit_store_dst(jd, iptr, d);
1504                         break;
1505
1506                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1507
1508                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1509                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1510                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1511                         /* implicit null-pointer check */
1512                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1513                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1514                         emit_store_dst(jd, iptr, d);
1515                         break;
1516
1517                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1518
1519                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1520                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1521                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1522                         /* implicit null-pointer check */
1523                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1524                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1525                         emit_store_dst(jd, iptr, d);
1526                         break;
1527
1528                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1529
1530                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1531                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1532                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1533                         /* implicit null-pointer check */
1534                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1535                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1536                         emit_store_dst(jd, iptr, d);
1537                         break;
1538
1539                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1540
1541                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1542                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1543                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1544                         /* implicit null-pointer check */
1545                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1546                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1547                         emit_store_dst(jd, iptr, d);
1548                         break;
1549
1550                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1551
1552                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1553                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1554                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1555                         /* implicit null-pointer check */
1556                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1557                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1558                         emit_store_dst(jd, iptr, d);
1559                         break;
1560
1561                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1562
1563                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1564                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1565                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1566                         /* implicit null-pointer check */
1567                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1568                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1569                         emit_store_dst(jd, iptr, d);
1570                         break;
1571
1572                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1573
1574                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1575                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1576                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1577                         /* implicit null-pointer check */
1578                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1579                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1580                         emit_store_dst(jd, iptr, d);
1581                         break;
1582
1583
1584                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1585
1586                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1587                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1588                         /* implicit null-pointer check */
1589                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1590                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1591                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1592                         break;
1593
1594                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1595
1596                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1597                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1598                         /* implicit null-pointer check */
1599                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1600                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1601                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1602                         break;
1603
1604                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1605
1606                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1607                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1608                         /* implicit null-pointer check */
1609                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1610                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1611                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1612                         break;
1613
1614                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1615
1616                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1617                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1618                         /* implicit null-pointer check */
1619                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1620                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1621                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1622                         break;
1623
1624                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1625
1626                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1627                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1628                         /* implicit null-pointer check */
1629                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1630                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1631                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1632                         break;
1633
1634                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1635
1636                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1637                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1638                         /* implicit null-pointer check */
1639                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1640                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1641                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1642                         break;
1643
1644                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1645
1646                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1647                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1648                         /* implicit null-pointer check */
1649                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1650                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1651                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1652                         break;
1653
1654                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1655
1656                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1657                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1658                         /* implicit null-pointer check */
1659                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1660                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1661
1662                         M_MOV(s1, REG_A0);
1663                         M_MOV(s3, REG_A1);
1664                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
1665                         M_CALL(REG_ITMP1);
1666                         emit_arraystore_check(cd, iptr);
1667
1668                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1669                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1670                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1671                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1672                         break;
1673
1674
1675                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1676
1677                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1678                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1679                         /* implicit null-pointer check */
1680                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1681                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1682                         break;
1683
1684                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1685
1686                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1687                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1688                         /* implicit null-pointer check */
1689                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1690                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1691                         break;
1692
1693                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1694
1695                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1696                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1697                         /* implicit null-pointer check */
1698                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1699                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1700                         break;
1701
1702                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1703
1704                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1705                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1706                         /* implicit null-pointer check */
1707                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1708                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1709                         break;
1710
1711                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1712
1713                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1714                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1715                         /* implicit null-pointer check */
1716                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1717
1718                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1719                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1720                         }
1721                         else {
1722                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1723                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1724                         }
1725                         break;
1726
1727                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1728
1729                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1730                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1731                         /* implicit null-pointer check */
1732                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1733                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1734                         break;
1735
1736
1737                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1738
1739                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1740                                 uf        = iptr->sx.s23.s3.uf;
1741                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1742                                 disp      = dseg_add_unique_address(cd, uf);
1743
1744 /*                              PROFILE_CYCLE_STOP; */
1745
1746                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1747
1748 /*                              PROFILE_CYCLE_START; */
1749                         }
1750                         else {
1751                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1752                                 fieldtype = fi->type;
1753                                 disp      = dseg_add_address(cd, fi->value);
1754
1755                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1756                                         PROFILE_CYCLE_STOP;
1757
1758                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1759                                                                                   fi->clazz, 0);
1760
1761                                         PROFILE_CYCLE_START;
1762                                 }
1763                         }
1764
1765                         /* This approach is much faster than moving the field
1766                            address inline into a register. */
1767
1768                         M_ALD(REG_ITMP1, RIP, disp);
1769
1770                         switch (fieldtype) {
1771                         case TYPE_INT:
1772                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1773                                 M_ILD(d, REG_ITMP1, 0);
1774                                 break;
1775                         case TYPE_LNG:
1776                         case TYPE_ADR:
1777                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1778                                 M_LLD(d, REG_ITMP1, 0);
1779                                 break;
1780                         case TYPE_FLT:
1781                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1782                                 M_FLD(d, REG_ITMP1, 0);
1783                                 break;
1784                         case TYPE_DBL:
1785                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1786                                 M_DLD(d, REG_ITMP1, 0);
1787                                 break;
1788                         }
1789                         emit_store_dst(jd, iptr, d);
1790                         break;
1791
1792                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1793
1794                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1795                                 uf        = iptr->sx.s23.s3.uf;
1796                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1797                                 disp      = dseg_add_unique_address(cd, uf);
1798
1799 /*                              PROFILE_CYCLE_STOP; */
1800
1801                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1802
1803 /*                              PROFILE_CYCLE_START; */
1804                         }
1805                         else {
1806                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1807                                 fieldtype = fi->type;
1808                                 disp      = dseg_add_address(cd, fi->value);
1809
1810                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1811                                         PROFILE_CYCLE_STOP;
1812
1813                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1814                                                                                   fi->clazz, 0);
1815
1816                                         PROFILE_CYCLE_START;
1817                                 }
1818                         }
1819
1820                         /* This approach is much faster than moving the field
1821                            address inline into a register. */
1822
1823                         M_ALD(REG_ITMP1, RIP, disp);
1824
1825                         switch (fieldtype) {
1826                         case TYPE_INT:
1827                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1828                                 M_IST(s1, REG_ITMP1, 0);
1829                                 break;
1830                         case TYPE_LNG:
1831                         case TYPE_ADR:
1832                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1833                                 M_LST(s1, REG_ITMP1, 0);
1834                                 break;
1835                         case TYPE_FLT:
1836                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1837                                 M_FST(s1, REG_ITMP1, 0);
1838                                 break;
1839                         case TYPE_DBL:
1840                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1841                                 M_DST(s1, REG_ITMP1, 0);
1842                                 break;
1843                         }
1844                         break;
1845
1846                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1847                                           /* val = value (in current instruction)     */
1848                                           /* following NOP)                           */
1849
1850                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1851                                 uf        = iptr->sx.s23.s3.uf;
1852                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1853                                 disp      = dseg_add_unique_address(cd, uf);
1854
1855 /*                              PROFILE_CYCLE_STOP; */
1856
1857                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1858
1859 /*                              PROFILE_CYCLE_START; */
1860                         }
1861                         else {
1862                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1863                                 fieldtype = fi->type;
1864                                 disp      = dseg_add_address(cd, fi->value);
1865
1866                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1867                                         PROFILE_CYCLE_STOP;
1868
1869                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1870                                                                                   fi->clazz, 0);
1871
1872                                         PROFILE_CYCLE_START;
1873                                 }
1874                         }
1875
1876                         /* This approach is much faster than moving the field
1877                            address inline into a register. */
1878
1879                         M_ALD(REG_ITMP1, RIP, disp);
1880
1881                         switch (fieldtype) {
1882                         case TYPE_INT:
1883                         case TYPE_FLT:
1884                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1885                                 break;
1886                         case TYPE_LNG:
1887                         case TYPE_ADR:
1888                         case TYPE_DBL:
1889                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1890                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1891                                 else {
1892                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1893                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1894                                 }
1895                                 break;
1896                         }
1897                         break;
1898
1899                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1900
1901                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1902
1903                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1904                                 uf        = iptr->sx.s23.s3.uf;
1905                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1906                                 disp      = 0;
1907
1908 /*                              PROFILE_CYCLE_STOP; */
1909
1910                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1911
1912 /*                              PROFILE_CYCLE_START; */
1913                         }
1914                         else {
1915                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1916                                 fieldtype = fi->type;
1917                                 disp      = fi->offset;
1918                         }
1919
1920                         /* implicit null-pointer check */
1921                         switch (fieldtype) {
1922                         case TYPE_INT:
1923                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1924                                 M_ILD32(d, s1, disp);
1925                                 break;
1926                         case TYPE_LNG:
1927                         case TYPE_ADR:
1928                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1929                                 M_LLD32(d, s1, disp);
1930                                 break;
1931                         case TYPE_FLT:
1932                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1933                                 M_FLD32(d, s1, disp);
1934                                 break;
1935                         case TYPE_DBL:
1936                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1937                                 M_DLD32(d, s1, disp);
1938                                 break;
1939                         }
1940                         emit_store_dst(jd, iptr, d);
1941                         break;
1942
1943                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1944
1945                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1946                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1947
1948                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1949                                 uf        = iptr->sx.s23.s3.uf;
1950                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1951                                 disp      = 0;
1952
1953 /*                              PROFILE_CYCLE_STOP; */
1954
1955                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1956
1957 /*                              PROFILE_CYCLE_START; */
1958                         }
1959                         else {
1960                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1961                                 fieldtype = fi->type;
1962                                 disp      = fi->offset;
1963                         }
1964
1965                         /* implicit null-pointer check */
1966                         switch (fieldtype) {
1967                         case TYPE_INT:
1968                                 M_IST32(s2, s1, disp);
1969                                 break;
1970                         case TYPE_LNG:
1971                         case TYPE_ADR:
1972                                 M_LST32(s2, s1, disp);
1973                                 break;
1974                         case TYPE_FLT:
1975                                 M_FST32(s2, s1, disp);
1976                                 break;
1977                         case TYPE_DBL:
1978                                 M_DST32(s2, s1, disp);
1979                                 break;
1980                         }
1981                         break;
1982
1983                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
1984                                           /* val = value (in current instruction)     */
1985                                           /* following NOP)                           */
1986
1987                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1988
1989                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1990                                 uf        = iptr->sx.s23.s3.uf;
1991                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1992                                 disp      = 0;
1993
1994 /*                              PROFILE_CYCLE_STOP; */
1995
1996                                 patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
1997
1998 /*                              PROFILE_CYCLE_START; */
1999                         }
2000                         else {
2001                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2002                                 fieldtype = fi->type;
2003                                 disp      = fi->offset;
2004                         }
2005
2006                         /* implicit null-pointer check */
2007                         switch (fieldtype) {
2008                         case TYPE_INT:
2009                         case TYPE_FLT:
2010                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2011                                 break;
2012                         case TYPE_LNG:
2013                         case TYPE_ADR:
2014                         case TYPE_DBL:
2015                                 /* XXX why no check for IS_IMM32? */
2016                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2017                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2018                                 break;
2019                         }
2020                         break;
2021
2022
2023                 /* branch operations **************************************************/
2024
2025                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2026
2027                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2028                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2029
2030                         PROFILE_CYCLE_STOP;
2031
2032 #ifdef ENABLE_VERIFIER
2033                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2034                                 uc = iptr->sx.s23.s2.uc;
2035
2036                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2037                         }
2038 #endif /* ENABLE_VERIFIER */
2039
2040                         M_CALL_IMM(0);                            /* passing exception pc */
2041                         M_POP(REG_ITMP2_XPC);
2042
2043                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2044                         M_JMP(REG_ITMP3);
2045                         break;
2046
2047                 case ICMD_GOTO:         /* ... ==> ...                                */
2048                 case ICMD_RET:
2049
2050                         emit_br(cd, iptr->dst.block);
2051                         ALIGNCODENOP;
2052                         break;
2053
2054                 case ICMD_JSR:          /* ... ==> ...                                */
2055
2056                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2057                         ALIGNCODENOP;
2058                         break;
2059
2060                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2061                 case ICMD_IFNONNULL:
2062
2063                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2064                         M_TEST(s1);
2065                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2066                         break;
2067
2068                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2069                 case ICMD_IFLT:
2070                 case ICMD_IFLE:
2071                 case ICMD_IFNE:
2072                 case ICMD_IFGT:
2073                 case ICMD_IFGE:
2074
2075                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2076                         M_ICMP_IMM(iptr->sx.val.i, s1);
2077                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2078                         break;
2079
2080                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2081                 case ICMD_IF_LNE:
2082                 case ICMD_IF_LLT:
2083                 case ICMD_IF_LGE:
2084                 case ICMD_IF_LGT:
2085                 case ICMD_IF_LLE:
2086
2087                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2088                         if (IS_IMM32(iptr->sx.val.l))
2089                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2090                         else {
2091                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2092                                 M_LCMP(REG_ITMP2, s1);
2093                         }
2094                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2095                         break;
2096
2097                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2098                 case ICMD_IF_ICMPNE:
2099                 case ICMD_IF_ICMPLT:
2100                 case ICMD_IF_ICMPGE:
2101                 case ICMD_IF_ICMPGT:
2102                 case ICMD_IF_ICMPLE:
2103
2104                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2105                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2106                         M_ICMP(s2, s1);
2107                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2108                         break;
2109
2110                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2111                 case ICMD_IF_ACMPNE:
2112
2113                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2114                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2115                         M_LCMP(s2, s1);
2116                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2117                         break;
2118
2119                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2120                 case ICMD_IF_LCMPNE:
2121                 case ICMD_IF_LCMPLT:
2122                 case ICMD_IF_LCMPGE:
2123                 case ICMD_IF_LCMPGT:
2124                 case ICMD_IF_LCMPLE:
2125
2126                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2127                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2128                         M_LCMP(s2, s1);
2129                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2130                         break;
2131
2132                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2133                 case ICMD_LRETURN:
2134
2135                         REPLACEMENT_POINT_RETURN(cd, iptr);
2136                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2137                         M_INTMOVE(s1, REG_RESULT);
2138                         goto nowperformreturn;
2139
2140                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2141
2142                         REPLACEMENT_POINT_RETURN(cd, iptr);
2143                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2144                         M_INTMOVE(s1, REG_RESULT);
2145
2146 #ifdef ENABLE_VERIFIER
2147                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2148                                 uc = iptr->sx.s23.s2.uc;
2149
2150                                 PROFILE_CYCLE_STOP;
2151
2152                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2153
2154                                 PROFILE_CYCLE_START;
2155                         }
2156 #endif /* ENABLE_VERIFIER */
2157                         goto nowperformreturn;
2158
2159                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2160                 case ICMD_DRETURN:
2161
2162                         REPLACEMENT_POINT_RETURN(cd, iptr);
2163                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2164                         M_FLTMOVE(s1, REG_FRESULT);
2165                         goto nowperformreturn;
2166
2167                 case ICMD_RETURN:      /* ...  ==> ...                                */
2168
2169                         REPLACEMENT_POINT_RETURN(cd, iptr);
2170
2171 nowperformreturn:
2172                         {
2173                         s4 i, p;
2174
2175                         p = cd->stackframesize;
2176
2177 #if !defined(NDEBUG)
2178                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2179                                 emit_verbosecall_exit(jd);
2180 #endif /* !defined(NDEBUG) */
2181
2182 #if defined(ENABLE_THREADS)
2183                         if (checksync && code_is_synchronized(code)) {
2184                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2185
2186                                 /* we need to save the proper return value */
2187                                 switch (iptr->opc) {
2188                                 case ICMD_IRETURN:
2189                                 case ICMD_ARETURN:
2190                                 case ICMD_LRETURN:
2191                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2192                                         break;
2193                                 case ICMD_FRETURN:
2194                                 case ICMD_DRETURN:
2195                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2196                                         break;
2197                                 }
2198
2199                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2200                                 M_CALL(REG_ITMP1);
2201
2202                                 /* and now restore the proper return value */
2203                                 switch (iptr->opc) {
2204                                 case ICMD_IRETURN:
2205                                 case ICMD_ARETURN:
2206                                 case ICMD_LRETURN:
2207                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2208                                         break;
2209                                 case ICMD_FRETURN:
2210                                 case ICMD_DRETURN:
2211                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2212                                         break;
2213                                 }
2214                         }
2215 #endif
2216
2217                         /* restore saved registers */
2218
2219                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2220                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2221                         }
2222                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2223                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2224                         }
2225
2226                         /* deallocate stack */
2227
2228                         if (cd->stackframesize)
2229                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2230
2231                         /* generate method profiling code */
2232
2233                         PROFILE_CYCLE_STOP;
2234
2235                         M_RET;
2236                         }
2237                         break;
2238
2239
2240                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2241                         {
2242                                 s4 i, l;
2243                                 branch_target_t *table;
2244
2245                                 table = iptr->dst.table;
2246
2247                                 l = iptr->sx.s23.s2.tablelow;
2248                                 i = iptr->sx.s23.s3.tablehigh;
2249
2250                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2251                                 M_INTMOVE(s1, REG_ITMP1);
2252
2253                                 if (l != 0)
2254                                         M_ISUB_IMM(l, REG_ITMP1);
2255
2256                                 /* number of targets */
2257                                 i = i - l + 1;
2258
2259                 /* range check */
2260
2261                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2262                                 emit_bugt(cd, table[0].block);
2263
2264                                 /* build jump table top down and use address of lowest entry */
2265
2266                                 table += i;
2267
2268                                 while (--i >= 0) {
2269                                         dseg_add_target(cd, table->block);
2270                                         --table;
2271                                 }
2272
2273                                 /* length of dataseg after last dseg_add_target is used
2274                                    by load */
2275
2276                                 M_MOV_IMM(0, REG_ITMP2);
2277                                 dseg_adddata(cd);
2278                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2279                                 M_JMP(REG_ITMP1);
2280                         }
2281                         break;
2282
2283
2284                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2285                         {
2286                                 s4 i;
2287                                 lookup_target_t *lookup;
2288
2289                                 lookup = iptr->dst.lookup;
2290
2291                                 i = iptr->sx.s23.s2.lookupcount;
2292
2293                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2294                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2295
2296                                 while (--i >= 0) {
2297                                         M_ICMP_IMM(lookup->value, s1);
2298                                         emit_beq(cd, lookup->target.block);
2299                                         lookup++;
2300                                 }
2301
2302                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2303                                 ALIGNCODENOP;
2304                         }
2305                         break;
2306
2307
2308                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2309
2310                         REPLACEMENT_POINT_FORGC_BUILTIN(cd, iptr);
2311
2312                         bte = iptr->sx.s23.s3.bte;
2313                         md  = bte->md;
2314                         goto gen_method;
2315
2316                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2317
2318                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2319                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2320                 case ICMD_INVOKEINTERFACE:
2321
2322                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2323
2324                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2325                                 lm = NULL;
2326                                 um = iptr->sx.s23.s3.um;
2327                                 md = um->methodref->parseddesc.md;
2328                         }
2329                         else {
2330                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2331                                 um = NULL;
2332                                 md = lm->parseddesc;
2333                         }
2334
2335 gen_method:
2336                         s3 = md->paramcount;
2337
2338                         MCODECHECK((20 * s3) + 128);
2339
2340                         /* copy arguments to registers or stack location */
2341
2342                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2343                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2344                                 d   = md->params[s3].regoff;
2345
2346                                 /* already preallocated (ARGVAR)? */
2347
2348                                 if (var->flags & PREALLOC)
2349                                         continue;
2350
2351                                 if (IS_INT_LNG_TYPE(var->type)) {
2352                                         if (!md->params[s3].inmemory) {
2353                                                 s1 = emit_load(jd, iptr, var, d);
2354                                                 M_INTMOVE(s1, d);
2355                                         }
2356                                         else {
2357                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2358                                                 M_LST(s1, REG_SP, d);
2359                                         }
2360                                 }
2361                                 else {
2362                                         if (!md->params[s3].inmemory) {
2363                                                 s1 = emit_load(jd, iptr, var, d);
2364                                                 M_FLTMOVE(s1, d);
2365                                         }
2366                                         else {
2367                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2368
2369                                                 if (IS_2_WORD_TYPE(var->type))
2370                                                         M_DST(s1, REG_SP, d);
2371                                                 else
2372                                                         M_FST(s1, REG_SP, d);
2373                                         }
2374                                 }
2375                         }
2376
2377                         /* generate method profiling code */
2378
2379                         PROFILE_CYCLE_STOP;
2380
2381                         switch (iptr->opc) {
2382                         case ICMD_BUILTIN:
2383                                 if (bte->stub == NULL) {
2384                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2385                                 }
2386                                 else {
2387                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2388                                 }
2389                                 M_CALL(REG_ITMP1);
2390                                 break;
2391
2392                         case ICMD_INVOKESPECIAL:
2393                                 emit_nullpointer_check(cd, iptr, REG_A0);
2394                                 /* fall through */
2395
2396                         case ICMD_INVOKESTATIC:
2397                                 if (lm == NULL) {
2398                                         disp = dseg_add_unique_address(cd, um);
2399
2400                                         patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
2401                                                                                   um, disp);
2402                                 }
2403                                 else {
2404                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2405                                 }
2406
2407                                 M_ALD(REG_ITMP2, RIP, disp);
2408                                 M_CALL(REG_ITMP2);
2409                                 break;
2410
2411                         case ICMD_INVOKEVIRTUAL:
2412                                 if (lm == NULL) {
2413                                         patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
2414
2415                                         s1 = 0;
2416                                 }
2417                                 else {
2418                                         s1 = OFFSET(vftbl_t, table[0]) +
2419                                                 sizeof(methodptr) * lm->vftblindex;
2420                                 }
2421
2422                                 /* implicit null-pointer check */
2423                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2424                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2425                                 M_CALL(REG_ITMP3);
2426                                 break;
2427
2428                         case ICMD_INVOKEINTERFACE:
2429                                 if (lm == NULL) {
2430                                         patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
2431
2432                                         s1 = 0;
2433                                         s2 = 0;
2434                                 }
2435                                 else {
2436                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2437                                                 sizeof(methodptr) * lm->clazz->index;
2438
2439                                         s2 = sizeof(methodptr) * (lm - lm->clazz->methods);
2440                                 }
2441
2442                                 /* implicit null-pointer check */
2443                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2444                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2445                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2446                                 M_CALL(REG_ITMP3);
2447                                 break;
2448                         }
2449
2450                         /* generate method profiling code */
2451
2452                         PROFILE_CYCLE_START;
2453
2454                         /* store size of call code in replacement point */
2455
2456                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2457                         REPLACEMENT_POINT_FORGC_BUILTIN_RETURN(cd, iptr);
2458
2459                         /* store return value */
2460
2461                         switch (md->returntype.type) {
2462                         case TYPE_INT:
2463                         case TYPE_LNG:
2464                         case TYPE_ADR:
2465                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2466                                 M_INTMOVE(REG_RESULT, s1);
2467                                 emit_store_dst(jd, iptr, s1);
2468                                 break;
2469                         case TYPE_FLT:
2470                         case TYPE_DBL:
2471                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2472                                 M_FLTMOVE(REG_FRESULT, s1);
2473                                 emit_store_dst(jd, iptr, s1);
2474                                 break;
2475                         default:
2476                                 /* TYPE_VOID */
2477                                 break;
2478                         }
2479                         break;
2480
2481
2482                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2483
2484                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2485                                 /* object type cast-check */
2486
2487                                 classinfo *super;
2488                                 s4         superindex;
2489
2490                                 s4 looptarget;
2491
2492                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2493                                         super      = NULL;
2494                                         superindex = 0;
2495                                 }
2496                                 else {
2497                                         super      = iptr->sx.s23.s3.c.cls;
2498                                         superindex = super->index;
2499                                 }
2500
2501                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2502
2503                                 /* if class is not resolved, check which code to call */
2504
2505                                 if (super == NULL) {
2506                                         M_TEST(s1);
2507                                         emit_label_beq(cd, BRANCH_LABEL_1);
2508
2509                                         patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2510                                                                                   iptr->sx.s23.s3.c.ref, 0);
2511
2512                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2513                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2514                                         emit_label_beq(cd, BRANCH_LABEL_2);
2515                                 }
2516
2517                                 /* interface checkcast code */
2518
2519                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2520                                         if (super != NULL) {
2521                                                 M_TEST(s1);
2522                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2523                                         }
2524
2525                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2526
2527                                         if (super == NULL) {
2528                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
2529                                                                                           iptr->sx.s23.s3.c.ref,
2530                                                                                           0);
2531                                         }
2532
2533                                         M_ILD32(REG_ITMP3,
2534                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2535                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2536                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2537
2538                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2539                                                         OFFSET(vftbl_t, interfacetable[0]) -
2540                                                         superindex * sizeof(methodptr*));
2541                                         M_TEST(REG_ITMP3);
2542                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2543
2544                                         if (super == NULL)
2545                                                 emit_label_br(cd, BRANCH_LABEL_4);
2546                                         else
2547                                                 emit_label(cd, BRANCH_LABEL_3);
2548                                 }
2549
2550                                 /* class checkcast code */
2551
2552                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2553                                         if (super == NULL) {
2554                                                 emit_label(cd, BRANCH_LABEL_2);
2555
2556                                                 cr   = iptr->sx.s23.s3.c.ref;
2557                                                 disp = dseg_add_unique_address(cd, cr);
2558
2559                                                 patcher_add_patch_ref(jd,
2560                                                                                           PATCHER_resolve_classref_to_vftbl,
2561                                                                                           cr, disp);
2562                                         }
2563                                         else {
2564                                                 M_TEST(s1);
2565                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2566
2567                                                 disp = dseg_add_address(cd, super->vftbl);
2568                                         }
2569
2570                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2571                                         M_ALD(REG_ITMP3, RIP, disp);
2572
2573                                         if (super == NULL || super->vftbl->subtype_depth >= DISPLAY_SIZE) {
2574                                         M_AADD_IMM(-32, REG_SP); /* need some stack */
2575                                         if (s1 == REG_ITMP1)
2576                                                 M_AST(REG_ITMP1, REG_SP, -8 + 32);
2577
2578                                         M_ALD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_offset));
2579
2580                                         *(cd->mcodeptr++) = 0x4d;
2581                                         *(cd->mcodeptr++) = 0x3b;
2582                                         *(cd->mcodeptr++) = 0x1c;
2583                                         *(cd->mcodeptr++) = 0x02;
2584                                         /* cmp (ITMP2, ITMP1, 1), ITMP3 */
2585
2586                                         emit_label_beq(cd, BRANCH_LABEL_6);  /* good */
2587
2588                                         M_LCMP_IMM(OFFSET(vftbl_t, subtype_display[DISPLAY_SIZE]), REG_ITMP1);
2589                                         emit_classcast_check(cd, iptr, BRANCH_NE, REG_ITMP3, s1);
2590
2591                                         M_AST(REG_ITMP3, REG_SP, -16 + 32);
2592                                         M_AST_IMM32(0, REG_SP, -24 + 32);
2593                                         M_ALD(REG_ITMP1, REG_ITMP2, OFFSET(vftbl_t, subtype_overflow));
2594                                         looptarget = cd->mcodeptr - cd->mcodebase;
2595
2596                                         M_ALD(REG_ITMP3, REG_SP, -24 + 32);
2597                                         M_ICMP_MEMBASE(REG_ITMP2, OFFSET(vftbl_t, subtype_overflow_length), REG_ITMP3);
2598                                         emit_label_bge(cd, BRANCH_LABEL_9);  /* throw */
2599
2600                                         *(cd->mcodeptr++) = 0x4e;
2601                                         *(cd->mcodeptr++) = 0x8b;
2602                                         *(cd->mcodeptr++) = 0x1c;
2603                                         *(cd->mcodeptr++) = 0xd8;
2604                                         /* movq (ITMP1, ITMP3, 8), ITMP3 */
2605
2606                                         M_LCMP_MEMBASE(REG_SP, -16 + 32, REG_ITMP3);
2607                                         emit_label_beq(cd, BRANCH_LABEL_7);  /* good */
2608
2609                                         M_LINC_MEMBASE(REG_SP, -24 + 32);
2610                                         M_JMP_IMM2(looptarget - (cd->mcodeptr - cd->mcodebase) - 2); /* 1 byte displacement */
2611
2612                                         emit_label(cd, BRANCH_LABEL_9);
2613                                         M_AADD_IMM(32, REG_SP); /* restore stack frame */
2614                                         M_ALD_MEM(REG_ITMP3, TRAP_ClassCastException);
2615
2616                                         emit_label(cd, BRANCH_LABEL_7);
2617
2618                                         emit_label(cd, BRANCH_LABEL_6);
2619
2620                                         if (s1 == REG_ITMP1)
2621                                                 M_ALD(REG_ITMP1, REG_SP, -8 + 32);
2622                                         M_AADD_IMM(32, REG_SP);
2623                                         }
2624                                         else {
2625                                                 assert(super->vftbl->subtype_offset < 0x80);
2626                                                 *(cd->mcodeptr++) = 0x4d;
2627                                                 *(cd->mcodeptr++) = 0x3b;
2628                                                 *(cd->mcodeptr++) = 0x5a;
2629                                                 *(cd->mcodeptr++) = super->vftbl->subtype_offset;
2630                                                 /* cmp off(ITMP1), ITMP2 */
2631
2632                                                 emit_classcast_check(cd, iptr, BRANCH_NE, REG_ITMP3, s1);
2633                                         }
2634
2635                                         if (super != NULL)
2636                                                 emit_label(cd, BRANCH_LABEL_5);
2637                                 }
2638
2639                                 if (super == NULL) {
2640                                         emit_label(cd, BRANCH_LABEL_1);
2641                                         emit_label(cd, BRANCH_LABEL_4);
2642                                 }
2643
2644                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2645                         }
2646                         else {
2647                                 /* array type cast-check */
2648
2649                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2650                                 M_INTMOVE(s1, REG_A0);
2651
2652                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2653                                         cr   = iptr->sx.s23.s3.c.ref;
2654                                         disp = dseg_add_unique_address(cd, cr);
2655
2656                                         patcher_add_patch_ref(jd,
2657                                                                                   PATCHER_resolve_classref_to_classinfo,
2658                                                                                   cr, disp);
2659                                 }
2660                                 else {
2661                                         disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2662                                 }
2663
2664                                 M_ALD(REG_A1, RIP, disp);
2665                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2666                                 M_CALL(REG_ITMP1);
2667
2668                                 /* s1 may have been destroyed over the function call */
2669                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2670                                 M_TEST(REG_RESULT);
2671                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2672
2673                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2674                         }
2675
2676                         M_INTMOVE(s1, d);
2677                         emit_store_dst(jd, iptr, d);
2678                         break;
2679
2680                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2681
2682                         {
2683                         classinfo *super;
2684                         s4         superindex;
2685
2686                         s4 looptarget;
2687
2688                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2689                                 super      = NULL;
2690                                 superindex = 0;
2691                         }
2692                         else {
2693                                 super      = iptr->sx.s23.s3.c.cls;
2694                                 superindex = super->index;
2695                         }
2696
2697                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2698                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2699
2700                         if (s1 == d) {
2701                                 M_INTMOVE(s1, REG_ITMP1);
2702                                 s1 = REG_ITMP1;
2703                         }
2704
2705                         M_CLR(d);
2706
2707                         /* if class is not resolved, check which code to call */
2708
2709                         if (super == NULL) {
2710                                 M_TEST(s1);
2711                                 emit_label_beq(cd, BRANCH_LABEL_1);
2712
2713                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2714                                                                           iptr->sx.s23.s3.c.ref, 0);
2715
2716                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2717                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2718                                 emit_label_beq(cd, BRANCH_LABEL_2);
2719                         }
2720
2721                         /* interface instanceof code */
2722
2723                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2724                                 if (super != NULL) {
2725                                         M_TEST(s1);
2726                                         emit_label_beq(cd, BRANCH_LABEL_3);
2727                                 }
2728
2729                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2730
2731                                 if (super == NULL) {
2732                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
2733                                                                                   iptr->sx.s23.s3.c.ref, 0);
2734                                 }
2735
2736                                 M_ILD32(REG_ITMP3,
2737                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2738                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2739
2740                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2741
2742                                 M_BLE(a);
2743                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2744                                                 OFFSET(vftbl_t, interfacetable[0]) -
2745                                                 superindex * sizeof(methodptr*));
2746                                 M_TEST(REG_ITMP1);
2747                                 M_SETNE(d);
2748
2749                                 if (super == NULL)
2750                                         emit_label_br(cd, BRANCH_LABEL_4);
2751                                 else
2752                                         emit_label(cd, BRANCH_LABEL_3);
2753                         }
2754
2755                         /* class instanceof code */
2756
2757                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2758                                 if (super == NULL) {
2759                                         emit_label(cd, BRANCH_LABEL_2);
2760
2761                                         cr   = iptr->sx.s23.s3.c.ref;
2762                                         disp = dseg_add_unique_address(cd, cr);
2763
2764                                         patcher_add_patch_ref(jd,
2765                                                                                   PATCHER_resolve_classref_to_vftbl,
2766                                                                                   cr, disp);
2767                                 }
2768                                 else {
2769                                         M_TEST(s1);
2770                                         emit_label_beq(cd, BRANCH_LABEL_5);
2771
2772                                         disp = dseg_add_address(cd, super->vftbl);
2773                                 }
2774
2775                                 M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2776                                 M_ALD(REG_ITMP3, RIP, disp);
2777
2778                                 if (super == NULL || super->vftbl->subtype_depth >= DISPLAY_SIZE) {
2779                                 M_AADD_IMM(-32, REG_SP); /* need some stack */
2780                                 M_ALD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_offset));
2781
2782                                 *(cd->mcodeptr++) = 0x4d;
2783                                 *(cd->mcodeptr++) = 0x3b;
2784                                 *(cd->mcodeptr++) = 0x1c;
2785                                 *(cd->mcodeptr++) = 0x02;
2786                                 /* cmp (ITMP2, ITMP1, 1), ITMP3 */
2787
2788                                 emit_label_bne(cd, BRANCH_LABEL_6);
2789                                 if (d == REG_ITMP2) {
2790                                         M_SETE(d);
2791                                         M_BSEXT(d, d);
2792                                 } else
2793                                         M_LINC(d);
2794                                 emit_label_br(cd, BRANCH_LABEL_7);    /* done, true */
2795
2796                                 emit_label(cd, BRANCH_LABEL_6);
2797
2798                                 M_LCMP_IMM(OFFSET(vftbl_t, subtype_display[DISPLAY_SIZE]), REG_ITMP1);
2799                                 emit_label_bne(cd, BRANCH_LABEL_6);   /* done, false */
2800
2801                                 M_AST(REG_ITMP3, REG_SP, -16 + 32);
2802                                 M_AST_IMM32(0, REG_SP, -24 + 32);
2803                                 M_ALD(REG_ITMP1, REG_ITMP2, OFFSET(vftbl_t, subtype_overflow));
2804                                 looptarget = cd->mcodeptr - cd->mcodebase;
2805
2806                                 M_ALD(REG_ITMP3, REG_SP, -24 + 32);
2807                                 M_ICMP_MEMBASE(REG_ITMP2, OFFSET(vftbl_t, subtype_overflow_length), REG_ITMP3);
2808                                 emit_label_bge(cd, BRANCH_LABEL_8);   /* done, false */
2809
2810                                 *(cd->mcodeptr++) = 0x4e;
2811                                 *(cd->mcodeptr++) = 0x8b;
2812                                 *(cd->mcodeptr++) = 0x1c;
2813                                 *(cd->mcodeptr++) = 0xd8;
2814                                 /* movq (ITMP1, ITMP3, 8), ITMP3 */
2815
2816                                 M_LCMP_MEMBASE(REG_SP, -16 + 32, REG_ITMP3);
2817                                 emit_label_bne(cd, BRANCH_LABEL_9);
2818                                 if (d == REG_ITMP2) {
2819                                         M_SETE(d);
2820                                         M_BSEXT(d, d);
2821                                 } else
2822                                         M_LINC(d);
2823                                 emit_label_br(cd, BRANCH_LABEL_10);   /* done, true */
2824                                 emit_label(cd, BRANCH_LABEL_9);
2825
2826                                 M_LINC_MEMBASE(REG_SP, -24 + 32);
2827                                 M_JMP_IMM2(looptarget - (cd->mcodeptr - cd->mcodebase) - 2); /* 1 byte displacement */
2828
2829                                 emit_label(cd, BRANCH_LABEL_8);
2830                                 emit_label(cd, BRANCH_LABEL_6);
2831
2832                                 if (d == REG_ITMP2)
2833                                         M_CLR(d);
2834
2835                                 emit_label(cd, BRANCH_LABEL_10);
2836                                 emit_label(cd, BRANCH_LABEL_7);
2837                                 M_AADD_IMM(32, REG_SP);
2838                                 }
2839                                 else {
2840                                         assert(super->vftbl->subtype_offset < 0x80);
2841                                         *(cd->mcodeptr++) = 0x4d;
2842                                         *(cd->mcodeptr++) = 0x3b;
2843                                         *(cd->mcodeptr++) = 0x5a;
2844                                         *(cd->mcodeptr++) = super->vftbl->subtype_offset;
2845                                         /* cmp off(ITMP1), ITMP2 */
2846
2847                                         M_SETE(d);
2848                                         if (d == REG_ITMP2)
2849                                                 M_BSEXT(d, d);
2850                                 }
2851
2852                                 if (super != NULL)
2853                                         emit_label(cd, BRANCH_LABEL_5);
2854                         }
2855
2856                         if (super == NULL) {
2857                                 emit_label(cd, BRANCH_LABEL_1);
2858                                 emit_label(cd, BRANCH_LABEL_4);
2859                         }
2860
2861                         emit_store_dst(jd, iptr, d);
2862                         }
2863                         break;
2864
2865                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2866
2867                         /* check for negative sizes and copy sizes to stack if necessary  */
2868
2869                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2870
2871                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2872
2873                                 /* copy SAVEDVAR sizes to stack */
2874                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2875
2876                                 /* Already Preallocated? */
2877                                 if (!(var->flags & PREALLOC)) {
2878                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2879                                         M_LST(s2, REG_SP, s1 * 8);
2880                                 }
2881                         }
2882
2883                         /* a0 = dimension count */
2884
2885                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2886
2887                         /* is a patcher function set? */
2888
2889                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2890                                 cr   = iptr->sx.s23.s3.c.ref;
2891                                 disp = dseg_add_unique_address(cd, cr);
2892
2893                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
2894                                                                           cr, disp);
2895                         }
2896                         else {
2897                                 disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2898                         }
2899
2900                         /* a1 = classinfo */
2901
2902                         M_ALD(REG_A1, RIP, disp);
2903
2904                         /* a2 = pointer to dimensions = stack pointer */
2905
2906                         M_MOV(REG_SP, REG_A2);
2907
2908                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2909                         M_CALL(REG_ITMP1);
2910
2911                         /* check for exception before result assignment */
2912
2913                         emit_exception_check(cd, iptr);
2914
2915                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2916                         M_INTMOVE(REG_RESULT, s1);
2917                         emit_store_dst(jd, iptr, s1);
2918                         break;
2919
2920                 default:
2921                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2922                                                                                    iptr->opc);
2923                         return false;
2924         } /* switch */
2925
2926         } /* for instruction */
2927
2928         MCODECHECK(512); /* XXX require a lower number? */
2929
2930         /* At the end of a basic block we may have to append some nops,
2931            because the patcher stub calling code might be longer than the
2932            actual instruction. So codepatching does not change the
2933            following block unintentionally. */
2934
2935         if (cd->mcodeptr < cd->lastmcodeptr) {
2936                 while (cd->mcodeptr < cd->lastmcodeptr) {
2937                         M_NOP;
2938                 }
2939         }
2940
2941         } /* if (bptr -> flags >= BBREACHED) */
2942         } /* for basic block */
2943
2944         /* Generate patcher traps. */
2945
2946         emit_patcher_traps(jd);
2947
2948         /* everything's ok */
2949
2950         return true;
2951 }
2952
2953
2954 /* codegen_emit_stub_native ****************************************************
2955
2956    Emits a stub routine which calls a native method.
2957
2958 *******************************************************************************/
2959
2960 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f, int skipparams)
2961 {
2962         methodinfo  *m;
2963         codeinfo    *code;
2964         codegendata *cd;
2965         methoddesc  *md;
2966         int          i, j;
2967         int          s1, s2;
2968         int          disp;
2969
2970         /* Sanity check. */
2971
2972         assert(f != NULL);
2973
2974         /* Get required compiler data. */
2975
2976         m    = jd->m;
2977         code = jd->code;
2978         cd   = jd->cd;
2979
2980         /* initialize variables */
2981
2982         md = m->parseddesc;
2983
2984         /* calculate stack frame size */
2985
2986         cd->stackframesize =
2987                 sizeof(stackframeinfo_t) / SIZEOF_VOID_P +
2988                 sizeof(localref_table) / SIZEOF_VOID_P +
2989                 md->paramcount +
2990                 (md->returntype.type == TYPE_VOID ? 0 : 1) +
2991                 nmd->memuse;
2992
2993         ALIGN_ODD(cd->stackframesize);              /* keep stack 16-byte aligned */
2994
2995         /* create method header */
2996
2997         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2998         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2999         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
3000         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
3001         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
3002
3003 #if defined(ENABLE_PROFILING)
3004         /* generate native method profiling code */
3005
3006         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3007                 /* count frequency */
3008
3009                 M_MOV_IMM(code, REG_ITMP3);
3010                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3011         }
3012 #endif
3013
3014         /* generate stub code */
3015
3016         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3017
3018 #if defined(ENABLE_GC_CACAO)
3019         /* Save callee saved integer registers in stackframeinfo (GC may
3020            need to recover them during a collection). */
3021
3022         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
3023                 OFFSET(stackframeinfo_t, intregs);
3024
3025         for (i = 0; i < INT_SAV_CNT; i++)
3026                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3027 #endif
3028
3029         /* save integer and float argument registers */
3030
3031         for (i = 0; i < md->paramcount; i++) {
3032                 if (!md->params[i].inmemory) {
3033                         s1 = md->params[i].regoff;
3034
3035                         switch (md->paramtypes[i].type) {
3036                         case TYPE_INT:
3037                         case TYPE_LNG:
3038                         case TYPE_ADR:
3039                                 M_LST(s1, REG_SP, i * 8);
3040                                 break;
3041                         case TYPE_FLT:
3042                         case TYPE_DBL:
3043                                 M_DST(s1, REG_SP, i * 8);
3044                                 break;
3045                         }
3046                 }
3047         }
3048
3049         /* create dynamic stack info */
3050
3051         M_MOV(REG_SP, REG_A0);
3052         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3053         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3054         M_CALL(REG_ITMP1);
3055
3056         /* remember class argument */
3057
3058         if (m->flags & ACC_STATIC)
3059                 M_MOV(REG_RESULT, REG_ITMP2);
3060
3061         /* restore integer and float argument registers */
3062
3063         for (i = 0; i < md->paramcount; i++) {
3064                 if (!md->params[i].inmemory) {
3065                         s1 = md->params[i].regoff;
3066
3067                         switch (md->paramtypes[i].type) {
3068                         case TYPE_INT:
3069                         case TYPE_LNG:
3070                         case TYPE_ADR:
3071                                 M_LLD(s1, REG_SP, i * 8);
3072                                 break;
3073                         case TYPE_FLT:
3074                         case TYPE_DBL:
3075                                 M_DLD(s1, REG_SP, i * 8);
3076                                 break;
3077                         }
3078                 }
3079         }
3080
3081         /* Copy or spill arguments to new locations. */
3082
3083         for (i = md->paramcount - 1, j = i + skipparams; i >= 0; i--, j--) {
3084                 s2 = nmd->params[j].regoff;
3085
3086                 switch (md->paramtypes[i].type) {
3087                 case TYPE_INT:
3088                 case TYPE_LNG:
3089                 case TYPE_ADR:
3090                         if (!md->params[i].inmemory) {
3091                                 s1 = md->params[i].regoff;
3092
3093                                 if (!nmd->params[j].inmemory)
3094                                         M_INTMOVE(s1, s2);
3095                                 else
3096                                         M_LST(s1, REG_SP, s2);
3097                         }
3098                         else {
3099                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3100                                 M_LLD(REG_ITMP1, REG_SP, s1);
3101                                 M_LST(REG_ITMP1, REG_SP, s2);
3102                         }
3103                         break;
3104                 case TYPE_FLT:
3105                         /* We only copy spilled float arguments, as the float
3106                            argument registers keep unchanged. */
3107
3108                         if (md->params[i].inmemory) {
3109                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3110
3111                                 M_FLD(REG_FTMP1, REG_SP, s1);
3112                                 M_FST(REG_FTMP1, REG_SP, s2);
3113                         }
3114                         break;
3115                 case TYPE_DBL:
3116                         if (md->params[i].inmemory) {
3117                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3118                                 M_DLD(REG_FTMP1, REG_SP, s1);
3119                                 M_DST(REG_FTMP1, REG_SP, s2);
3120                         }
3121                         break;
3122                 }
3123         }
3124
3125         /* Handle native Java methods. */
3126
3127         if (m->flags & ACC_NATIVE) {
3128                 /* put class into second argument register */
3129
3130                 if (m->flags & ACC_STATIC)
3131                         M_MOV(REG_ITMP2, REG_A1);
3132
3133                 /* put env into first argument register */
3134
3135                 M_MOV_IMM(_Jv_env, REG_A0);
3136         }
3137
3138         /* Call the native function. */
3139
3140         disp = dseg_add_functionptr(cd, f);
3141         M_ALD(REG_ITMP1, RIP, disp);
3142         M_CALL(REG_ITMP1);
3143
3144         /* save return value */
3145
3146         switch (md->returntype.type) {
3147         case TYPE_INT:
3148         case TYPE_LNG:
3149         case TYPE_ADR:
3150                 switch (md->returntype.decltype) {
3151                 case PRIMITIVETYPE_BOOLEAN:
3152                         M_BZEXT(REG_RESULT, REG_RESULT);
3153                         break;
3154                 case PRIMITIVETYPE_BYTE:
3155                         M_BSEXT(REG_RESULT, REG_RESULT);
3156                         break;
3157                 case PRIMITIVETYPE_CHAR:
3158                         M_CZEXT(REG_RESULT, REG_RESULT);
3159                         break;
3160                 case PRIMITIVETYPE_SHORT:
3161                         M_SSEXT(REG_RESULT, REG_RESULT);
3162                         break;
3163                 }
3164                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3165                 break;
3166         case TYPE_FLT:
3167         case TYPE_DBL:
3168                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3169                 break;
3170         case TYPE_VOID:
3171                 break;
3172         }
3173
3174         /* remove native stackframe info */
3175
3176         M_MOV(REG_SP, REG_A0);
3177         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3178         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3179         M_CALL(REG_ITMP1);
3180         M_MOV(REG_RESULT, REG_ITMP3);
3181
3182         /* restore return value */
3183
3184         switch (md->returntype.type) {
3185         case TYPE_INT:
3186         case TYPE_LNG:
3187         case TYPE_ADR:
3188                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3189                 break;
3190         case TYPE_FLT:
3191         case TYPE_DBL:
3192                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3193                 break;
3194         case TYPE_VOID:
3195                 break;
3196         }
3197
3198 #if defined(ENABLE_GC_CACAO)
3199         /* Restore callee saved integer registers from stackframeinfo (GC
3200            might have modified them during a collection). */
3201
3202         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
3203                 OFFSET(stackframeinfo_t, intregs);
3204
3205         for (i = 0; i < INT_SAV_CNT; i++)
3206                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3207 #endif
3208
3209         /* remove stackframe */
3210
3211         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3212
3213         /* test for exception */
3214
3215         M_TEST(REG_ITMP3);
3216         M_BNE(1);
3217         M_RET;
3218
3219         /* handle exception */
3220
3221         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3222         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3223         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3224
3225         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3226         M_JMP(REG_ITMP3);
3227 }
3228
3229
3230 /*
3231  * These are local overrides for various environment variables in Emacs.
3232  * Please do not remove this and leave it at the end of the file, where
3233  * Emacs will automagically detect them.
3234  * ---------------------------------------------------------------------
3235  * Local variables:
3236  * mode: c
3237  * indent-tabs-mode: t
3238  * c-basic-offset: 4
3239  * tab-width: 4
3240  * End:
3241  * vim:noexpandtab:sw=4:ts=4:
3242  */