src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25 */
  26
  27
  28 #include "config.h"
  29
  30 #include <assert.h>
  31 #include <stdio.h>
  32 #include <stdint.h>
  33
  34 #include "vm/types.h"
  35
  36 #include "md-abi.h"
  37
  38 #include "vm/jit/x86_64/arch.h"
  39 #include "vm/jit/x86_64/codegen.h"
  40 #include "vm/jit/x86_64/emit.h"
  41
  42 #include "mm/memory.h"
  43
  44 #include "native/jni.h"
  45 #include "native/localref.h"
  46 #include "native/native.h"
  47
  48 #include "threads/lock-common.h"
  49
  50 #include "vm/builtin.h"
  51 #include "vm/exceptions.h"
  52 #include "vm/global.h"
  53 #include "vm/primitive.h"
  54 #include "vm/stringlocal.h"
  55 #include "vm/vm.h"
  56
  57 #include "vm/jit/abi.h"
  58 #include "vm/jit/asmpart.h"
  59 #include "vm/jit/code.h"
  60 #include "vm/jit/codegen-common.h"
  61 #include "vm/jit/dseg.h"
  62 #include "vm/jit/emit-common.h"
  63 #include "vm/jit/jit.h"
  64 #include "vm/jit/linenumbertable.h"
  65 #include "vm/jit/methodheader.h"
  66 #include "vm/jit/parse.h"
  67 #include "vm/jit/patcher-common.h"
  68 #include "vm/jit/reg.h"
  69 #include "vm/jit/replace.h"
  70 #include "vm/jit/stacktrace.h"
  71 #include "vm/jit/trap.h"
  72
  73 #if defined(ENABLE_LSRA)
  74 # include "vm/jit/allocator/lsra.h"
  75 #endif
  76
  77 #include "vmcore/loader.h"
  78 #include "vmcore/options.h"
  79 #include "vmcore/statistics.h"
  80
  81
  82 /* codegen_emit ****************************************************************
  83
  84    Generates machine code.
  85
  86 *******************************************************************************/
  87
  88 bool codegen_emit(jitdata *jd)
  89 {
  90         methodinfo         *m;
  91         codeinfo           *code;
  92         codegendata        *cd;
  93         registerdata       *rd;
  94         s4                  len, s1, s2, s3, d, disp;
  95         u2                  currentline;
  96         ptrint              a;
  97         varinfo            *var, *dst;
  98         basicblock         *bptr;
  99         instruction        *iptr;
 100         constant_classref  *cr;
 101         unresolved_class   *uc;
 102         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 103         unresolved_method  *um;
 104         builtintable_entry *bte;
 105         methoddesc         *md;
 106         fieldinfo          *fi;
 107         unresolved_field   *uf;
 108         s4                  fieldtype;
 109         s4                 varindex;
 110
 111         /* get required compiler data */
 112
 113         m    = jd->m;
 114         code = jd->code;
 115         cd   = jd->cd;
 116         rd   = jd->rd;
 117
 118         /* prevent compiler warnings */
 119
 120         d   = 0;
 121         lm  = NULL;
 122         um  = NULL;
 123         bte = NULL;
 124
 125         {
 126         s4 i, p, t, l;
 127         s4 savedregs_num;
 128
 129         savedregs_num = 0;
 130
 131         /* space to save used callee saved registers */
 132
 133         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 134         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 135
 136         cd->stackframesize = rd->memuse + savedregs_num;
 137
 138 #if defined(ENABLE_THREADS)
 139         /* space to save argument of monitor_enter */
 140
 141         if (checksync && code_is_synchronized(code))
 142                 cd->stackframesize++;
 143 #endif
 144
 145         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 146            native code e.g. libc or jni (alignment problems with
 147            movaps). */
 148
 149         if (!code_is_leafmethod(code) || opt_verbosecall)
 150                 cd->stackframesize |= 0x1;
 151
 152         /* create method header */
 153
 154         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 155         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 156
 157         code->synchronizedoffset = rd->memuse * 8;
 158
 159         if (code_is_leafmethod(code))
 160                 (void) dseg_add_unique_s4(cd, 1);                  /* IsLeaf          */
 161         else
 162                 (void) dseg_add_unique_s4(cd, 0);                  /* IsLeaf          */
 163
 164         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 165         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 166
 167 #if defined(ENABLE_PROFILING)
 168         /* generate method profiling code */
 169
 170         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 171                 /* count frequency */
 172
 173                 M_MOV_IMM(code, REG_ITMP3);
 174                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 175
 176                 PROFILE_CYCLE_START;
 177         }
 178 #endif
 179
 180         /* create stack frame (if necessary) */
 181
 182         if (cd->stackframesize)
 183                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 184
 185         /* save used callee saved registers */
 186
 187         p = cd->stackframesize;
 188         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 189                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 190         }
 191         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 192                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 193         }
 194
 195         /* take arguments out of register or stack frame */
 196
 197         md = m->parseddesc;
 198
 199         for (p = 0, l = 0; p < md->paramcount; p++) {
 200                 t = md->paramtypes[p].type;
 201
 202                 varindex = jd->local_map[l * 5 + t];
 203
 204                 l++;
 205                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 206                         l++;
 207
 208                 if (varindex == UNUSED)
 209                         continue;
 210
 211                 var = VAR(varindex);
 212
 213                 s1 = md->params[p].regoff;
 214
 215                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 216                         if (!md->params[p].inmemory) {           /* register arguments    */
 217                                 if (!IS_INMEMORY(var->flags))
 218                                         M_INTMOVE(s1, var->vv.regoff);
 219                                 else
 220                                     M_LST(s1, REG_SP, var->vv.regoff);
 221                         }
 222                         else {                                 /* stack arguments       */
 223                                 if (!IS_INMEMORY(var->flags))
 224                                         /* + 8 for return address */
 225                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 226                                 else
 227                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 228                         }
 229                 }
 230                 else {                                     /* floating args         */
 231                         if (!md->params[p].inmemory) {           /* register arguments    */
 232                                 if (!IS_INMEMORY(var->flags))
 233                                         M_FLTMOVE(s1, var->vv.regoff);
 234                                 else
 235                                         M_DST(s1, REG_SP, var->vv.regoff);
 236                         }
 237                         else {                                 /* stack arguments       */
 238                                 if (!IS_INMEMORY(var->flags))
 239                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 240                                 else
 241                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 242                         }
 243                 }
 244         }
 245
 246         /* save monitorenter argument */
 247
 248 #if defined(ENABLE_THREADS)
 249         if (checksync && code_is_synchronized(code)) {
 250                 /* stack offset for monitor argument */
 251
 252                 s1 = rd->memuse;
 253
 254                 if (opt_verbosecall) {
 255                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 256
 257                         for (p = 0; p < INT_ARG_CNT; p++)
 258                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 259
 260                         for (p = 0; p < FLT_ARG_CNT; p++)
 261                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 262
 263                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 264                 }
 265
 266                 /* decide which monitor enter function to call */
 267
 268                 if (m->flags & ACC_STATIC) {
 269                         M_MOV_IMM(&m->clazz->object.header, REG_A0);
 270                 }
 271                 else {
 272                         M_TEST(REG_A0);
 273                         M_BNE(8);
 274                         M_ALD_MEM(REG_A0, TRAP_NullPointerException);
 275                 }
 276
 277                 M_AST(REG_A0, REG_SP, s1 * 8);
 278                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 279                 M_CALL(REG_ITMP1);
 280
 281                 if (opt_verbosecall) {
 282                         for (p = 0; p < INT_ARG_CNT; p++)
 283                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 284
 285                         for (p = 0; p < FLT_ARG_CNT; p++)
 286                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 287
 288                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 289                 }
 290         }
 291 #endif
 292
 293 #if !defined(NDEBUG)
 294         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 295                 emit_verbosecall_enter(jd);
 296 #endif /* !defined(NDEBUG) */
 297
 298         }
 299
 300         /* end of header generation */
 301
 302         /* create replacement points */
 303
 304         REPLACEMENT_POINTS_INIT(cd, jd);
 305
 306         /* walk through all basic blocks */
 307
 308         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 309
 310                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 311
 312                 if (bptr->flags >= BBREACHED) {
 313
 314                 /* branch resolving */
 315
 316                 codegen_resolve_branchrefs(cd, bptr);
 317
 318                 /* handle replacement points */
 319
 320                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 321
 322                 /* copy interface registers to their destination */
 323
 324                 len = bptr->indepth;
 325                 MCODECHECK(512);
 326
 327 #if defined(ENABLE_PROFILING)
 328                 /* generate basicblock profiling code */
 329
 330                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 331                         /* count frequency */
 332
 333                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 334                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 335
 336                         /* if this is an exception handler, start profiling again */
 337
 338                         if (bptr->type == BBTYPE_EXH)
 339                                 PROFILE_CYCLE_START;
 340                 }
 341 #endif
 342
 343 #if defined(ENABLE_LSRA)
 344                 if (opt_lsra) {
 345                         while (len) {
 346                                 len--;
 347                                 src = bptr->invars[len];
 348                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 349                                         if (bptr->type == BBTYPE_EXH) {
 350 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 351                                                 if (!IS_INMEMORY(src->flags))
 352                                                         d= src->vv.regoff;
 353                                                 else
 354                                                         d=REG_ITMP1;
 355                                                 M_INTMOVE(REG_ITMP1, d);
 356                                                 emit_store(jd, NULL, src, d);
 357                                         }
 358                                 }
 359                         }
 360
 361                 } else {
 362 #endif
 363
 364                 while (len) {
 365                         len--;
 366                         var = VAR(bptr->invars[len]);
 367                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 368                                 if (bptr->type == BBTYPE_EXH) {
 369                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 370                                         M_INTMOVE(REG_ITMP1, d);
 371                                         emit_store(jd, NULL, var, d);
 372                                 }
 373                         }
 374                         else {
 375                                 assert((var->flags & INOUT));
 376                         }
 377                 }
 378 #if defined(ENABLE_LSRA)
 379                 }
 380 #endif
 381                 /* walk through all instructions */
 382
 383                 len = bptr->icount;
 384                 currentline = 0;
 385
 386                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 387                         if (iptr->line != currentline) {
 388                                 linenumbertable_list_entry_add(cd, iptr->line);
 389                                 currentline = iptr->line;
 390                         }
 391
 392                         MCODECHECK(1024);                         /* 1KB should be enough */
 393
 394                 switch (iptr->opc) {
 395                 case ICMD_NOP:        /* ...  ==> ...                                 */
 396                 case ICMD_POP:        /* ..., value  ==> ...                          */
 397                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 398                         break;
 399
 400                 case ICMD_INLINE_START:
 401
 402                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 403                         break;
 404
 405                 case ICMD_INLINE_BODY:
 406
 407                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 408                         linenumbertable_list_entry_add_inline_start(cd, iptr);
 409                         linenumbertable_list_entry_add(cd, iptr->line);
 410                         break;
 411
 412                 case ICMD_INLINE_END:
 413
 414                         linenumbertable_list_entry_add_inline_end(cd, iptr);
 415                         linenumbertable_list_entry_add(cd, iptr->line);
 416                         break;
 417
 418                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 419
 420                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 421                         emit_nullpointer_check(cd, iptr, s1);
 422                         break;
 423
 424                 /* constant operations ************************************************/
 425
 426                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 427
 428                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 429                         ICONST(d, iptr->sx.val.i);
 430                         emit_store_dst(jd, iptr, d);
 431                         break;
 432
 433                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 434
 435                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 436                         LCONST(d, iptr->sx.val.l);
 437                         emit_store_dst(jd, iptr, d);
 438                         break;
 439
 440                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 441
 442                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 443                         disp = dseg_add_float(cd, iptr->sx.val.f);
 444                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 445                         emit_store_dst(jd, iptr, d);
 446                         break;
 447
 448                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 449
 450                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 451                         disp = dseg_add_double(cd, iptr->sx.val.d);
 452                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 453                         emit_store_dst(jd, iptr, d);
 454                         break;
 455
 456                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 457
 458                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 459
 460                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 461                                 cr   = iptr->sx.val.c.ref;
 462                                 disp = dseg_add_unique_address(cd, cr);
 463
 464 /*                              PROFILE_CYCLE_STOP; */
 465
 466                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
 467                                                                           cr, disp);
 468
 469 /*                              PROFILE_CYCLE_START; */
 470
 471                                 M_ALD(d, RIP, disp);
 472                         }
 473                         else {
 474                                 if (iptr->sx.val.anyptr == 0) {
 475                                         M_CLR(d);
 476                                 }
 477                                 else {
 478                                         disp = dseg_add_address(cd, iptr->sx.val.anyptr);
 479                                         M_ALD(d, RIP, disp);
 480                                 }
 481                         }
 482                         emit_store_dst(jd, iptr, d);
 483                         break;
 484
 485
 486                 /* load/store/copy/move operations ************************************/
 487
 488                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 489                 case ICMD_ALOAD:      /* s1 = local variable                          */
 490                 case ICMD_LLOAD:
 491                 case ICMD_FLOAD:
 492                 case ICMD_DLOAD:
 493                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 494                 case ICMD_LSTORE:
 495                 case ICMD_FSTORE:
 496                 case ICMD_DSTORE:
 497                 case ICMD_COPY:
 498                 case ICMD_MOVE:
 499
 500                         emit_copy(jd, iptr);
 501                         break;
 502
 503                 case ICMD_ASTORE:
 504                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 505                                 emit_copy(jd, iptr);
 506                         break;
 507
 508                 /* integer operations *************************************************/
 509
 510                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 511
 512                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 513                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 514                         M_INTMOVE(s1, d);
 515                         M_INEG(d);
 516                         emit_store_dst(jd, iptr, d);
 517                         break;
 518
 519                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 520
 521                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 522                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 523                         M_INTMOVE(s1, d);
 524                         M_LNEG(d);
 525                         emit_store_dst(jd, iptr, d);
 526                         break;
 527
 528                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 529
 530                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 531                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 532                         M_ISEXT(s1, d);
 533                         emit_store_dst(jd, iptr, d);
 534                         break;
 535
 536                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 537
 538                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 539                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 540                         M_IMOV(s1, d);
 541                         emit_store_dst(jd, iptr, d);
 542                         break;
 543
 544                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 545
 546                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 547                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 548                         M_BSEXT(s1, d);
 549                         emit_store_dst(jd, iptr, d);
 550                         break;
 551
 552                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 553
 554                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 555                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 556                         M_CZEXT(s1, d);
 557                         emit_store_dst(jd, iptr, d);
 558                         break;
 559
 560                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 561
 562                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 563                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 564                         M_SSEXT(s1, d);
 565                         emit_store_dst(jd, iptr, d);
 566                         break;
 567
 568
 569                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 570
 571                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 572                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 573                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 574                         if (s2 == d)
 575                                 M_IADD(s1, d);
 576                         else {
 577                                 M_INTMOVE(s1, d);
 578                                 M_IADD(s2, d);
 579                         }
 580                         emit_store_dst(jd, iptr, d);
 581                         break;
 582
 583                 case ICMD_IINC:
 584                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 585                                       /* sx.val.i = constant                             */
 586
 587                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 588                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 589
 590                         /* Using inc and dec is not faster than add (tested with
 591                            sieve). */
 592
 593                         M_INTMOVE(s1, d);
 594                         M_IADD_IMM(iptr->sx.val.i, d);
 595                         emit_store_dst(jd, iptr, d);
 596                         break;
 597
 598                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 599
 600                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 601                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 602                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 603                         if (s2 == d)
 604                                 M_LADD(s1, d);
 605                         else {
 606                                 M_INTMOVE(s1, d);
 607                                 M_LADD(s2, d);
 608                         }
 609                         emit_store_dst(jd, iptr, d);
 610                         break;
 611
 612                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 613                                       /* sx.val.l = constant                             */
 614
 615                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 616                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 617                         M_INTMOVE(s1, d);
 618                         if (IS_IMM32(iptr->sx.val.l))
 619                                 M_LADD_IMM(iptr->sx.val.l, d);
 620                         else {
 621                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 622                                 M_LADD(REG_ITMP2, d);
 623                         }
 624                         emit_store_dst(jd, iptr, d);
 625                         break;
 626
 627                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 628
 629                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 630                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 631                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 632                         if (s2 == d) {
 633                                 M_INTMOVE(s1, REG_ITMP1);
 634                                 M_ISUB(s2, REG_ITMP1);
 635                                 M_INTMOVE(REG_ITMP1, d);
 636                         } else {
 637                                 M_INTMOVE(s1, d);
 638                                 M_ISUB(s2, d);
 639                         }
 640                         emit_store_dst(jd, iptr, d);
 641                         break;
 642
 643                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 644                                       /* sx.val.i = constant                             */
 645
 646                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 647                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 648                         M_INTMOVE(s1, d);
 649                         M_ISUB_IMM(iptr->sx.val.i, d);
 650                         emit_store_dst(jd, iptr, d);
 651                         break;
 652
 653                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 654
 655                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 656                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 657                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 658                         if (s2 == d) {
 659                                 M_INTMOVE(s1, REG_ITMP1);
 660                                 M_LSUB(s2, REG_ITMP1);
 661                                 M_INTMOVE(REG_ITMP1, d);
 662                         } else {
 663                                 M_INTMOVE(s1, d);
 664                                 M_LSUB(s2, d);
 665                         }
 666                         emit_store_dst(jd, iptr, d);
 667                         break;
 668
 669                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 670                                       /* sx.val.l = constant                             */
 671
 672                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 673                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 674                         M_INTMOVE(s1, d);
 675                         if (IS_IMM32(iptr->sx.val.l))
 676                                 M_LSUB_IMM(iptr->sx.val.l, d);
 677                         else {
 678                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 679                                 M_LSUB(REG_ITMP2, d);
 680                         }
 681                         emit_store_dst(jd, iptr, d);
 682                         break;
 683
 684                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 685
 686                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 687                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 688                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 689                         if (s2 == d)
 690                                 M_IMUL(s1, d);
 691                         else {
 692                                 M_INTMOVE(s1, d);
 693                                 M_IMUL(s2, d);
 694                         }
 695                         emit_store_dst(jd, iptr, d);
 696                         break;
 697
 698                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 699                                       /* sx.val.i = constant                             */
 700
 701                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 702                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 703                         if (iptr->sx.val.i == 2) {
 704                                 M_INTMOVE(s1, d);
 705                                 M_ISLL_IMM(1, d);
 706                         } else
 707                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 708                         emit_store_dst(jd, iptr, d);
 709                         break;
 710
 711                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 712
 713                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 714                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 715                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 716                         if (s2 == d)
 717                                 M_LMUL(s1, d);
 718                         else {
 719                                 M_INTMOVE(s1, d);
 720                                 M_LMUL(s2, d);
 721                         }
 722                         emit_store_dst(jd, iptr, d);
 723                         break;
 724
 725                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 726                                       /* sx.val.l = constant                             */
 727
 728                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 729                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 730                         if (IS_IMM32(iptr->sx.val.l))
 731                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 732                         else {
 733                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 734                                 M_INTMOVE(s1, d);
 735                                 M_LMUL(REG_ITMP2, d);
 736                         }
 737                         emit_store_dst(jd, iptr, d);
 738                         break;
 739
 740                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 741
 742                         s1 = emit_load_s1(jd, iptr, RAX);
 743                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 744                         d = codegen_reg_of_dst(jd, iptr, RAX);
 745
 746                         M_INTMOVE(s1, RAX);
 747                         M_INTMOVE(s2, REG_ITMP3);
 748                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 749
 750                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 751
 752                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 753                         M_BNE(4 + 6);
 754                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 755                         M_BEQ(1 + 3);                                          /* 6 bytes */
 756
 757                         emit_cltd(cd);                                         /* 1 byte  */
 758                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 759
 760                         M_INTMOVE(RAX, d);
 761                         emit_store_dst(jd, iptr, d);
 762                         dst = VAROP(iptr->dst);
 763                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 764                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 765                         break;
 766
 767                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 768
 769                         s1 = emit_load_s1(jd, iptr, RAX);
 770                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 771                         d = codegen_reg_of_dst(jd, iptr, RDX);
 772
 773                         M_INTMOVE(s1, RAX);
 774                         M_INTMOVE(s2, REG_ITMP3);
 775                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 776
 777                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 778
 779                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 780                         M_BNE(3 + 4 + 6);
 781                         M_CLR(RDX);                                            /* 3 bytes */
 782                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 783                         M_BEQ(1 + 3);                                          /* 6 bytes */
 784
 785                         emit_cltd(cd);                                         /* 1 byte  */
 786                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 787
 788                         M_INTMOVE(RDX, d);
 789                         emit_store_dst(jd, iptr, d);
 790                         dst = VAROP(iptr->dst);
 791                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 792                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 793                         break;
 794
 795                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 796                                       /* sx.val.i = constant                             */
 797
 798                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 799                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 800                         M_INTMOVE(s1, REG_ITMP1);
 801                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 802                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 803                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 804                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 805                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 806                         emit_store_dst(jd, iptr, d);
 807                         break;
 808
 809                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 810                                       /* sx.val.i = constant                             */
 811
 812                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 813                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 814                         M_INTMOVE(s1, REG_ITMP1);
 815                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 816                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 817                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 818                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 819                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 820                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 821                         emit_store_dst(jd, iptr, d);
 822                         break;
 823
 824
 825                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 826
 827                         s1 = emit_load_s1(jd, iptr, RAX);
 828                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 829                         d = codegen_reg_of_dst(jd, iptr, RAX);
 830
 831                         M_INTMOVE(s1, RAX);
 832                         M_INTMOVE(s2, REG_ITMP3);
 833                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 834
 835                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 836
 837                         /* check as described in jvm spec */
 838                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 839                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 840                         M_BNE(4 + 6);
 841                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 842                         M_BEQ(2 + 3);                                          /* 6 bytes */
 843
 844                         emit_cqto(cd);                                         /* 2 bytes */
 845                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 846
 847                         M_INTMOVE(RAX, d);
 848                         emit_store_dst(jd, iptr, d);
 849                         dst = VAROP(iptr->dst);
 850                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 851                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 852                         break;
 853
 854                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 855
 856                         s1 = emit_load_s1(jd, iptr, RAX);
 857                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 858                         d = codegen_reg_of_dst(jd, iptr, RDX);
 859
 860                         M_INTMOVE(s1, RAX);
 861                         M_INTMOVE(s2, REG_ITMP3);
 862                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 863
 864                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 865
 866                         /* check as described in jvm spec */
 867                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 868                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 869                         M_BNE(3 + 4 + 6);
 870                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 871                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 872                         M_BEQ(2 + 3);                                          /* 6 bytes */
 873
 874                         emit_cqto(cd);                                         /* 2 bytes */
 875                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 876
 877                         M_INTMOVE(RDX, d);
 878                         emit_store_dst(jd, iptr, d);
 879                         dst = VAROP(iptr->dst);
 880                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 881                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 882                         break;
 883
 884                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 885                                       /* sx.val.i = constant                             */
 886
 887                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 888                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 889                         M_INTMOVE(s1, REG_ITMP1);
 890                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 891                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 892                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 893                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 894                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 895                         emit_store_dst(jd, iptr, d);
 896                         break;
 897
 898                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 899                                       /* sx.val.l = constant                             */
 900
 901                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 902                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 903                         M_INTMOVE(s1, REG_ITMP1);
 904                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 905                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 906                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 907                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 908                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 909                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 910                         emit_store_dst(jd, iptr, d);
 911                         break;
 912
 913                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 914
 915                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 916                         emit_ishift(jd, SHIFT_SHL, iptr);
 917                         break;
 918
 919                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 920                                       /* sx.val.i = constant                             */
 921
 922                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 923                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 924                         M_INTMOVE(s1, d);
 925                         M_ISLL_IMM(iptr->sx.val.i, d);
 926                         emit_store_dst(jd, iptr, d);
 927                         break;
 928
 929                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 930
 931                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 932                         emit_ishift(jd, SHIFT_SAR, iptr);
 933                         break;
 934
 935                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 936                                       /* sx.val.i = constant                             */
 937
 938                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 939                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 940                         M_INTMOVE(s1, d);
 941                         M_ISRA_IMM(iptr->sx.val.i, d);
 942                         emit_store_dst(jd, iptr, d);
 943                         break;
 944
 945                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 946
 947                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 948                         emit_ishift(jd, SHIFT_SHR, iptr);
 949                         break;
 950
 951                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 952                                       /* sx.val.i = constant                             */
 953
 954                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 955                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 956                         M_INTMOVE(s1, d);
 957                         M_ISRL_IMM(iptr->sx.val.i, d);
 958                         emit_store_dst(jd, iptr, d);
 959                         break;
 960
 961                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 962
 963                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 964                         emit_lshift(jd, SHIFT_SHL, iptr);
 965                         break;
 966
 967         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 968                                           /* sx.val.i = constant                             */
 969
 970                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 971                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 972                         M_INTMOVE(s1, d);
 973                         M_LSLL_IMM(iptr->sx.val.i, d);
 974                         emit_store_dst(jd, iptr, d);
 975                         break;
 976
 977                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 978
 979                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 980                         emit_lshift(jd, SHIFT_SAR, iptr);
 981                         break;
 982
 983                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 984                                       /* sx.val.i = constant                             */
 985
 986                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 987                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 988                         M_INTMOVE(s1, d);
 989                         M_LSRA_IMM(iptr->sx.val.i, d);
 990                         emit_store_dst(jd, iptr, d);
 991                         break;
 992
 993                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 994
 995                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 996                         emit_lshift(jd, SHIFT_SHR, iptr);
 997                         break;
 998
 999                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1000                                       /* sx.val.l = constant                             */
1001
1002                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1003                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1004                         M_INTMOVE(s1, d);
1005                         M_LSRL_IMM(iptr->sx.val.i, d);
1006                         emit_store_dst(jd, iptr, d);
1007                         break;
1008
1009                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1010
1011                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1012                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1013                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1014                         if (s2 == d)
1015                                 M_IAND(s1, d);
1016                         else {
1017                                 M_INTMOVE(s1, d);
1018                                 M_IAND(s2, d);
1019                         }
1020                         emit_store_dst(jd, iptr, d);
1021                         break;
1022
1023                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1024                                       /* sx.val.i = constant                             */
1025
1026                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1027                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1028                         M_INTMOVE(s1, d);
1029                         M_IAND_IMM(iptr->sx.val.i, d);
1030                         emit_store_dst(jd, iptr, d);
1031                         break;
1032
1033                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1034
1035                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1036                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1037                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1038                         if (s2 == d)
1039                                 M_LAND(s1, d);
1040                         else {
1041                                 M_INTMOVE(s1, d);
1042                                 M_LAND(s2, d);
1043                         }
1044                         emit_store_dst(jd, iptr, d);
1045                         break;
1046
1047                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1048                                       /* sx.val.l = constant                             */
1049
1050                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1051                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1052                         M_INTMOVE(s1, d);
1053                         if (IS_IMM32(iptr->sx.val.l))
1054                                 M_LAND_IMM(iptr->sx.val.l, d);
1055                         else {
1056                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1057                                 M_LAND(REG_ITMP2, d);
1058                         }
1059                         emit_store_dst(jd, iptr, d);
1060                         break;
1061
1062                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1063
1064                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1065                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1066                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1067                         if (s2 == d)
1068                                 M_IOR(s1, d);
1069                         else {
1070                                 M_INTMOVE(s1, d);
1071                                 M_IOR(s2, d);
1072                         }
1073                         emit_store_dst(jd, iptr, d);
1074                         break;
1075
1076                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1077                                       /* sx.val.i = constant                             */
1078
1079                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1080                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1081                         M_INTMOVE(s1, d);
1082                         M_IOR_IMM(iptr->sx.val.i, d);
1083                         emit_store_dst(jd, iptr, d);
1084                         break;
1085
1086                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1087
1088                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1089                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1090                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1091                         if (s2 == d)
1092                                 M_LOR(s1, d);
1093                         else {
1094                                 M_INTMOVE(s1, d);
1095                                 M_LOR(s2, d);
1096                         }
1097                         emit_store_dst(jd, iptr, d);
1098                         break;
1099
1100                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1101                                       /* sx.val.l = constant                             */
1102
1103                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1104                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1105                         M_INTMOVE(s1, d);
1106                         if (IS_IMM32(iptr->sx.val.l))
1107                                 M_LOR_IMM(iptr->sx.val.l, d);
1108                         else {
1109                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1110                                 M_LOR(REG_ITMP2, d);
1111                         }
1112                         emit_store_dst(jd, iptr, d);
1113                         break;
1114
1115                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1116
1117                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1118                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1119                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1120                         if (s2 == d)
1121                                 M_IXOR(s1, d);
1122                         else {
1123                                 M_INTMOVE(s1, d);
1124                                 M_IXOR(s2, d);
1125                         }
1126                         emit_store_dst(jd, iptr, d);
1127                         break;
1128
1129                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1130                                       /* sx.val.i = constant                             */
1131
1132                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1133                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1134                         M_INTMOVE(s1, d);
1135                         M_IXOR_IMM(iptr->sx.val.i, d);
1136                         emit_store_dst(jd, iptr, d);
1137                         break;
1138
1139                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1140
1141                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1142                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1143                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1144                         if (s2 == d)
1145                                 M_LXOR(s1, d);
1146                         else {
1147                                 M_INTMOVE(s1, d);
1148                                 M_LXOR(s2, d);
1149                         }
1150                         emit_store_dst(jd, iptr, d);
1151                         break;
1152
1153                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1154                                       /* sx.val.l = constant                             */
1155
1156                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1157                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1158                         M_INTMOVE(s1, d);
1159                         if (IS_IMM32(iptr->sx.val.l))
1160                                 M_LXOR_IMM(iptr->sx.val.l, d);
1161                         else {
1162                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1163                                 M_LXOR(REG_ITMP2, d);
1164                         }
1165                         emit_store_dst(jd, iptr, d);
1166                         break;
1167
1168
1169                 /* floating operations ************************************************/
1170
1171                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1172
1173                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1174                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1175                         disp = dseg_add_s4(cd, 0x80000000);
1176                         M_FLTMOVE(s1, d);
1177                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1178                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1179                         emit_store_dst(jd, iptr, d);
1180                         break;
1181
1182                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1183
1184                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1185                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1186                         disp = dseg_add_s8(cd, 0x8000000000000000);
1187                         M_FLTMOVE(s1, d);
1188                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1189                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1190                         emit_store_dst(jd, iptr, d);
1191                         break;
1192
1193                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1194
1195                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1196                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1197                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1198                         if (s2 == d)
1199                                 M_FADD(s1, d);
1200                         else {
1201                                 M_FLTMOVE(s1, d);
1202                                 M_FADD(s2, d);
1203                         }
1204                         emit_store_dst(jd, iptr, d);
1205                         break;
1206
1207                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1208
1209                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1210                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1211                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1212                         if (s2 == d)
1213                                 M_DADD(s1, d);
1214                         else {
1215                                 M_FLTMOVE(s1, d);
1216                                 M_DADD(s2, d);
1217                         }
1218                         emit_store_dst(jd, iptr, d);
1219                         break;
1220
1221                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1222
1223                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1224                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1225                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1226                         if (s2 == d) {
1227                                 M_FLTMOVE(s2, REG_FTMP2);
1228                                 s2 = REG_FTMP2;
1229                         }
1230                         M_FLTMOVE(s1, d);
1231                         M_FSUB(s2, d);
1232                         emit_store_dst(jd, iptr, d);
1233                         break;
1234
1235                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1236
1237                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1238                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1239                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1240                         if (s2 == d) {
1241                                 M_FLTMOVE(s2, REG_FTMP2);
1242                                 s2 = REG_FTMP2;
1243                         }
1244                         M_FLTMOVE(s1, d);
1245                         M_DSUB(s2, d);
1246                         emit_store_dst(jd, iptr, d);
1247                         break;
1248
1249                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1250
1251                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1252                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1253                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1254                         if (s2 == d)
1255                                 M_FMUL(s1, d);
1256                         else {
1257                                 M_FLTMOVE(s1, d);
1258                                 M_FMUL(s2, d);
1259                         }
1260                         emit_store_dst(jd, iptr, d);
1261                         break;
1262
1263                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1264
1265                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1266                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1267                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1268                         if (s2 == d)
1269                                 M_DMUL(s1, d);
1270                         else {
1271                                 M_FLTMOVE(s1, d);
1272                                 M_DMUL(s2, d);
1273                         }
1274                         emit_store_dst(jd, iptr, d);
1275                         break;
1276
1277                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1278
1279                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1280                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1281                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1282                         if (s2 == d) {
1283                                 M_FLTMOVE(s2, REG_FTMP2);
1284                                 s2 = REG_FTMP2;
1285                         }
1286                         M_FLTMOVE(s1, d);
1287                         M_FDIV(s2, d);
1288                         emit_store_dst(jd, iptr, d);
1289                         break;
1290
1291                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1292
1293                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1294                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1295                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1296                         if (s2 == d) {
1297                                 M_FLTMOVE(s2, REG_FTMP2);
1298                                 s2 = REG_FTMP2;
1299                         }
1300                         M_FLTMOVE(s1, d);
1301                         M_DDIV(s2, d);
1302                         emit_store_dst(jd, iptr, d);
1303                         break;
1304
1305                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1306
1307                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1308                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1309                         M_CVTIF(s1, d);
1310                         emit_store_dst(jd, iptr, d);
1311                         break;
1312
1313                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1314
1315                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1316                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1317                         M_CVTID(s1, d);
1318                         emit_store_dst(jd, iptr, d);
1319                         break;
1320
1321                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1322
1323                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1324                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1325                         M_CVTLF(s1, d);
1326                         emit_store_dst(jd, iptr, d);
1327                         break;
1328
1329                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1330
1331                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1332                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1333                         M_CVTLD(s1, d);
1334                         emit_store_dst(jd, iptr, d);
1335                         break;
1336
1337                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1338
1339                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1340                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1341                         M_CVTFI(s1, d);
1342                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1343                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1344                                 ((REG_RESULT == d) ? 0 : 3);
1345                         M_BNE(disp);
1346                         M_FLTMOVE(s1, REG_FTMP1);
1347                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1348                         M_CALL(REG_ITMP2);
1349                         M_INTMOVE(REG_RESULT, d);
1350                         emit_store_dst(jd, iptr, d);
1351                         break;
1352
1353                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1354
1355                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1356                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1357                         M_CVTDI(s1, d);
1358                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1359                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1360                                 ((REG_RESULT == d) ? 0 : 3);
1361                         M_BNE(disp);
1362                         M_FLTMOVE(s1, REG_FTMP1);
1363                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1364                         M_CALL(REG_ITMP2);
1365                         M_INTMOVE(REG_RESULT, d);
1366                         emit_store_dst(jd, iptr, d);
1367                         break;
1368
1369                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1370
1371                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1372                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1373                         M_CVTFL(s1, d);
1374                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1375                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1376                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1377                                 ((REG_RESULT == d) ? 0 : 3);
1378                         M_BNE(disp);
1379                         M_FLTMOVE(s1, REG_FTMP1);
1380                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1381                         M_CALL(REG_ITMP2);
1382                         M_INTMOVE(REG_RESULT, d);
1383                         emit_store_dst(jd, iptr, d);
1384                         break;
1385
1386                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1387
1388                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1389                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1390                         M_CVTDL(s1, d);
1391                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1392                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1393                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1394                                 ((REG_RESULT == d) ? 0 : 3);
1395                         M_BNE(disp);
1396                         M_FLTMOVE(s1, REG_FTMP1);
1397                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1398                         M_CALL(REG_ITMP2);
1399                         M_INTMOVE(REG_RESULT, d);
1400                         emit_store_dst(jd, iptr, d);
1401                         break;
1402
1403                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1404
1405                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1406                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1407                         M_CVTFD(s1, d);
1408                         emit_store_dst(jd, iptr, d);
1409                         break;
1410
1411                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1412
1413                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1414                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1415                         M_CVTDF(s1, d);
1416                         emit_store_dst(jd, iptr, d);
1417                         break;
1418
1419                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1420                                           /* == => 0, < => 1, > => -1 */
1421
1422                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1423                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1424                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1425                         M_CLR(d);
1426                         M_MOV_IMM(1, REG_ITMP1);
1427                         M_MOV_IMM(-1, REG_ITMP2);
1428                         emit_ucomiss_reg_reg(cd, s1, s2);
1429                         M_CMOVULT(REG_ITMP1, d);
1430                         M_CMOVUGT(REG_ITMP2, d);
1431                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1432                         emit_store_dst(jd, iptr, d);
1433                         break;
1434
1435                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1436                                           /* == => 0, < => 1, > => -1 */
1437
1438                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1439                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1440                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1441                         M_CLR(d);
1442                         M_MOV_IMM(1, REG_ITMP1);
1443                         M_MOV_IMM(-1, REG_ITMP2);
1444                         emit_ucomiss_reg_reg(cd, s1, s2);
1445                         M_CMOVULT(REG_ITMP1, d);
1446                         M_CMOVUGT(REG_ITMP2, d);
1447                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1448                         emit_store_dst(jd, iptr, d);
1449                         break;
1450
1451                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1452                                           /* == => 0, < => 1, > => -1 */
1453
1454                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1455                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1456                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1457                         M_CLR(d);
1458                         M_MOV_IMM(1, REG_ITMP1);
1459                         M_MOV_IMM(-1, REG_ITMP2);
1460                         emit_ucomisd_reg_reg(cd, s1, s2);
1461                         M_CMOVULT(REG_ITMP1, d);
1462                         M_CMOVUGT(REG_ITMP2, d);
1463                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1464                         emit_store_dst(jd, iptr, d);
1465                         break;
1466
1467                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1468                                           /* == => 0, < => 1, > => -1 */
1469
1470                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1471                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1472                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1473                         M_CLR(d);
1474                         M_MOV_IMM(1, REG_ITMP1);
1475                         M_MOV_IMM(-1, REG_ITMP2);
1476                         emit_ucomisd_reg_reg(cd, s1, s2);
1477                         M_CMOVULT(REG_ITMP1, d);
1478                         M_CMOVUGT(REG_ITMP2, d);
1479                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1480                         emit_store_dst(jd, iptr, d);
1481                         break;
1482
1483
1484                 /* memory operations **************************************************/
1485
1486                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1487
1488                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1489                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1490                         /* implicit null-pointer check */
1491                         M_ILD(d, s1, OFFSET(java_array_t, size));
1492                         emit_store_dst(jd, iptr, d);
1493                         break;
1494
1495                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1496
1497                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1498                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1499                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1500                         /* implicit null-pointer check */
1501                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1502                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1503                         emit_store_dst(jd, iptr, d);
1504                         break;
1505
1506                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1507
1508                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1509                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1510                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1511                         /* implicit null-pointer check */
1512                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1513                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1514                         emit_store_dst(jd, iptr, d);
1515                         break;
1516
1517                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1518
1519                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1520                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1521                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1522                         /* implicit null-pointer check */
1523                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1524                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1525                         emit_store_dst(jd, iptr, d);
1526                         break;
1527
1528                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1529
1530                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1531                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1532                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1533                         /* implicit null-pointer check */
1534                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1535                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1536                         emit_store_dst(jd, iptr, d);
1537                         break;
1538
1539                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1540
1541                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1542                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1543                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1544                         /* implicit null-pointer check */
1545                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1546                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1547                         emit_store_dst(jd, iptr, d);
1548                         break;
1549
1550                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1551
1552                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1553                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1554                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1555                         /* implicit null-pointer check */
1556                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1557                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1558                         emit_store_dst(jd, iptr, d);
1559                         break;
1560
1561                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1562
1563                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1564                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1565                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1566                         /* implicit null-pointer check */
1567                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1568                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1569                         emit_store_dst(jd, iptr, d);
1570                         break;
1571
1572                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1573
1574                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1575                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1576                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1577                         /* implicit null-pointer check */
1578                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1579                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1580                         emit_store_dst(jd, iptr, d);
1581                         break;
1582
1583
1584                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1585
1586                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1587                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1588                         /* implicit null-pointer check */
1589                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1590                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1591                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1592                         break;
1593
1594                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1595
1596                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1597                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1598                         /* implicit null-pointer check */
1599                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1600                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1601                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1602                         break;
1603
1604                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1605
1606                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1607                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1608                         /* implicit null-pointer check */
1609                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1610                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1611                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1612                         break;
1613
1614                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1615
1616                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1617                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1618                         /* implicit null-pointer check */
1619                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1620                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1621                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1622                         break;
1623
1624                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1625
1626                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1627                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1628                         /* implicit null-pointer check */
1629                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1630                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1631                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1632                         break;
1633
1634                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1635
1636                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1637                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1638                         /* implicit null-pointer check */
1639                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1640                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1641                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1642                         break;
1643
1644                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1645
1646                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1647                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1648                         /* implicit null-pointer check */
1649                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1650                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1651                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1652                         break;
1653
1654                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1655
1656                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1657                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1658                         /* implicit null-pointer check */
1659                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1660                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1661
1662                         M_MOV(s1, REG_A0);
1663                         M_MOV(s3, REG_A1);
1664                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
1665                         M_CALL(REG_ITMP1);
1666                         emit_arraystore_check(cd, iptr);
1667
1668                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1669                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1670                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1671                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1672                         break;
1673
1674
1675                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1676
1677                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1678                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1679                         /* implicit null-pointer check */
1680                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1681                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1682                         break;
1683
1684                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1685
1686                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1687                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1688                         /* implicit null-pointer check */
1689                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1690                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1691                         break;
1692
1693                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1694
1695                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1696                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1697                         /* implicit null-pointer check */
1698                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1699                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1700                         break;
1701
1702                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1703
1704                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1705                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1706                         /* implicit null-pointer check */
1707                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1708                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1709                         break;
1710
1711                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1712
1713                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1714                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1715                         /* implicit null-pointer check */
1716                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1717
1718                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1719                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1720                         }
1721                         else {
1722                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1723                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1724                         }
1725                         break;
1726
1727                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1728
1729                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1730                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1731                         /* implicit null-pointer check */
1732                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1733                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1734                         break;
1735
1736
1737                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1738
1739                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1740                                 uf        = iptr->sx.s23.s3.uf;
1741                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1742                                 disp      = dseg_add_unique_address(cd, uf);
1743
1744 /*                              PROFILE_CYCLE_STOP; */
1745
1746                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1747
1748 /*                              PROFILE_CYCLE_START; */
1749                         }
1750                         else {
1751                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1752                                 fieldtype = fi->type;
1753                                 disp      = dseg_add_address(cd, fi->value);
1754
1755                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1756                                         PROFILE_CYCLE_STOP;
1757
1758                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1759                                                                                   fi->clazz, 0);
1760
1761                                         PROFILE_CYCLE_START;
1762                                 }
1763                         }
1764
1765                         /* This approach is much faster than moving the field
1766                            address inline into a register. */
1767
1768                         M_ALD(REG_ITMP1, RIP, disp);
1769
1770                         switch (fieldtype) {
1771                         case TYPE_INT:
1772                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1773                                 M_ILD(d, REG_ITMP1, 0);
1774                                 break;
1775                         case TYPE_LNG:
1776                         case TYPE_ADR:
1777                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1778                                 M_LLD(d, REG_ITMP1, 0);
1779                                 break;
1780                         case TYPE_FLT:
1781                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1782                                 M_FLD(d, REG_ITMP1, 0);
1783                                 break;
1784                         case TYPE_DBL:
1785                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1786                                 M_DLD(d, REG_ITMP1, 0);
1787                                 break;
1788                         }
1789                         emit_store_dst(jd, iptr, d);
1790                         break;
1791
1792                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1793
1794                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1795                                 uf        = iptr->sx.s23.s3.uf;
1796                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1797                                 disp      = dseg_add_unique_address(cd, uf);
1798
1799 /*                              PROFILE_CYCLE_STOP; */
1800
1801                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1802
1803 /*                              PROFILE_CYCLE_START; */
1804                         }
1805                         else {
1806                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1807                                 fieldtype = fi->type;
1808                                 disp      = dseg_add_address(cd, fi->value);
1809
1810                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1811                                         PROFILE_CYCLE_STOP;
1812
1813                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1814                                                                                   fi->clazz, 0);
1815
1816                                         PROFILE_CYCLE_START;
1817                                 }
1818                         }
1819
1820                         /* This approach is much faster than moving the field
1821                            address inline into a register. */
1822
1823                         M_ALD(REG_ITMP1, RIP, disp);
1824
1825                         switch (fieldtype) {
1826                         case TYPE_INT:
1827                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1828                                 M_IST(s1, REG_ITMP1, 0);
1829                                 break;
1830                         case TYPE_LNG:
1831                         case TYPE_ADR:
1832                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1833                                 M_LST(s1, REG_ITMP1, 0);
1834                                 break;
1835                         case TYPE_FLT:
1836                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1837                                 M_FST(s1, REG_ITMP1, 0);
1838                                 break;
1839                         case TYPE_DBL:
1840                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1841                                 M_DST(s1, REG_ITMP1, 0);
1842                                 break;
1843                         }
1844                         break;
1845
1846                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1847                                           /* val = value (in current instruction)     */
1848                                           /* following NOP)                           */
1849
1850                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1851                                 uf        = iptr->sx.s23.s3.uf;
1852                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1853                                 disp      = dseg_add_unique_address(cd, uf);
1854
1855 /*                              PROFILE_CYCLE_STOP; */
1856
1857                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1858
1859 /*                              PROFILE_CYCLE_START; */
1860                         }
1861                         else {
1862                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1863                                 fieldtype = fi->type;
1864                                 disp      = dseg_add_address(cd, fi->value);
1865
1866                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1867                                         PROFILE_CYCLE_STOP;
1868
1869                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1870                                                                                   fi->clazz, 0);
1871
1872                                         PROFILE_CYCLE_START;
1873                                 }
1874                         }
1875
1876                         /* This approach is much faster than moving the field
1877                            address inline into a register. */
1878
1879                         M_ALD(REG_ITMP1, RIP, disp);
1880
1881                         switch (fieldtype) {
1882                         case TYPE_INT:
1883                         case TYPE_FLT:
1884                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1885                                 break;
1886                         case TYPE_LNG:
1887                         case TYPE_ADR:
1888                         case TYPE_DBL:
1889                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1890                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1891                                 else {
1892                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1893                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1894                                 }
1895                                 break;
1896                         }
1897                         break;
1898
1899                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1900
1901                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1902
1903                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1904                                 uf        = iptr->sx.s23.s3.uf;
1905                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1906                                 disp      = 0;
1907
1908 /*                              PROFILE_CYCLE_STOP; */
1909
1910                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1911
1912 /*                              PROFILE_CYCLE_START; */
1913                         }
1914                         else {
1915                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1916                                 fieldtype = fi->type;
1917                                 disp      = fi->offset;
1918                         }
1919
1920                         /* implicit null-pointer check */
1921                         switch (fieldtype) {
1922                         case TYPE_INT:
1923                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1924                                 M_ILD32(d, s1, disp);
1925                                 break;
1926                         case TYPE_LNG:
1927                         case TYPE_ADR:
1928                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1929                                 M_LLD32(d, s1, disp);
1930                                 break;
1931                         case TYPE_FLT:
1932                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1933                                 M_FLD32(d, s1, disp);
1934                                 break;
1935                         case TYPE_DBL:
1936                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1937                                 M_DLD32(d, s1, disp);
1938                                 break;
1939                         }
1940                         emit_store_dst(jd, iptr, d);
1941                         break;
1942
1943                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1944
1945                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1946                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1947
1948                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1949                                 uf        = iptr->sx.s23.s3.uf;
1950                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1951                                 disp      = 0;
1952
1953 /*                              PROFILE_CYCLE_STOP; */
1954
1955                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1956
1957 /*                              PROFILE_CYCLE_START; */
1958                         }
1959                         else {
1960                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1961                                 fieldtype = fi->type;
1962                                 disp      = fi->offset;
1963                         }
1964
1965                         /* implicit null-pointer check */
1966                         switch (fieldtype) {
1967                         case TYPE_INT:
1968                                 M_IST32(s2, s1, disp);
1969                                 break;
1970                         case TYPE_LNG:
1971                         case TYPE_ADR:
1972                                 M_LST32(s2, s1, disp);
1973                                 break;
1974                         case TYPE_FLT:
1975                                 M_FST32(s2, s1, disp);
1976                                 break;
1977                         case TYPE_DBL:
1978                                 M_DST32(s2, s1, disp);
1979                                 break;
1980                         }
1981                         break;
1982
1983                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
1984                                           /* val = value (in current instruction)     */
1985                                           /* following NOP)                           */
1986
1987                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1988
1989                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1990                                 uf        = iptr->sx.s23.s3.uf;
1991                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1992                                 disp      = 0;
1993
1994 /*                              PROFILE_CYCLE_STOP; */
1995
1996                                 patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
1997
1998 /*                              PROFILE_CYCLE_START; */
1999                         }
2000                         else {
2001                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2002                                 fieldtype = fi->type;
2003                                 disp      = fi->offset;
2004                         }
2005
2006                         /* implicit null-pointer check */
2007                         switch (fieldtype) {
2008                         case TYPE_INT:
2009                         case TYPE_FLT:
2010                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2011                                 break;
2012                         case TYPE_LNG:
2013                         case TYPE_ADR:
2014                         case TYPE_DBL:
2015                                 /* XXX why no check for IS_IMM32? */
2016                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2017                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2018                                 break;
2019                         }
2020                         break;
2021
2022
2023                 /* branch operations **************************************************/
2024
2025                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2026
2027                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2028                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2029
2030                         PROFILE_CYCLE_STOP;
2031
2032 #ifdef ENABLE_VERIFIER
2033                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2034                                 uc = iptr->sx.s23.s2.uc;
2035
2036                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2037                         }
2038 #endif /* ENABLE_VERIFIER */
2039
2040                         M_CALL_IMM(0);                            /* passing exception pc */
2041                         M_POP(REG_ITMP2_XPC);
2042
2043                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2044                         M_JMP(REG_ITMP3);
2045                         break;
2046
2047                 case ICMD_GOTO:         /* ... ==> ...                                */
2048                 case ICMD_RET:
2049
2050                         emit_br(cd, iptr->dst.block);
2051                         ALIGNCODENOP;
2052                         break;
2053
2054                 case ICMD_JSR:          /* ... ==> ...                                */
2055
2056                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2057                         ALIGNCODENOP;
2058                         break;
2059
2060                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2061                 case ICMD_IFNONNULL:
2062
2063                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2064                         M_TEST(s1);
2065                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2066                         break;
2067
2068                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2069                 case ICMD_IFLT:
2070                 case ICMD_IFLE:
2071                 case ICMD_IFNE:
2072                 case ICMD_IFGT:
2073                 case ICMD_IFGE:
2074
2075                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2076                         M_ICMP_IMM(iptr->sx.val.i, s1);
2077                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2078                         break;
2079
2080                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2081                 case ICMD_IF_LNE:
2082                 case ICMD_IF_LLT:
2083                 case ICMD_IF_LGE:
2084                 case ICMD_IF_LGT:
2085                 case ICMD_IF_LLE:
2086
2087                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2088                         if (IS_IMM32(iptr->sx.val.l))
2089                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2090                         else {
2091                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2092                                 M_LCMP(REG_ITMP2, s1);
2093                         }
2094                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2095                         break;
2096
2097                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2098                 case ICMD_IF_ICMPNE:
2099                 case ICMD_IF_ICMPLT:
2100                 case ICMD_IF_ICMPGE:
2101                 case ICMD_IF_ICMPGT:
2102                 case ICMD_IF_ICMPLE:
2103
2104                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2105                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2106                         M_ICMP(s2, s1);
2107                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2108                         break;
2109
2110                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2111                 case ICMD_IF_ACMPNE:
2112
2113                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2114                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2115                         M_LCMP(s2, s1);
2116                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2117                         break;
2118
2119                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2120                 case ICMD_IF_LCMPNE:
2121                 case ICMD_IF_LCMPLT:
2122                 case ICMD_IF_LCMPGE:
2123                 case ICMD_IF_LCMPGT:
2124                 case ICMD_IF_LCMPLE:
2125
2126                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2127                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2128                         M_LCMP(s2, s1);
2129                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2130                         break;
2131
2132                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2133                 case ICMD_LRETURN:
2134
2135                         REPLACEMENT_POINT_RETURN(cd, iptr);
2136                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2137                         M_INTMOVE(s1, REG_RESULT);
2138                         goto nowperformreturn;
2139
2140                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2141
2142                         REPLACEMENT_POINT_RETURN(cd, iptr);
2143                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2144                         M_INTMOVE(s1, REG_RESULT);
2145
2146 #ifdef ENABLE_VERIFIER
2147                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2148                                 uc = iptr->sx.s23.s2.uc;
2149
2150                                 PROFILE_CYCLE_STOP;
2151
2152                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2153
2154                                 PROFILE_CYCLE_START;
2155                         }
2156 #endif /* ENABLE_VERIFIER */
2157                         goto nowperformreturn;
2158
2159                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2160                 case ICMD_DRETURN:
2161
2162                         REPLACEMENT_POINT_RETURN(cd, iptr);
2163                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2164                         M_FLTMOVE(s1, REG_FRESULT);
2165                         goto nowperformreturn;
2166
2167                 case ICMD_RETURN:      /* ...  ==> ...                                */
2168
2169                         REPLACEMENT_POINT_RETURN(cd, iptr);
2170
2171 nowperformreturn:
2172                         {
2173                         s4 i, p;
2174
2175                         p = cd->stackframesize;
2176
2177 #if !defined(NDEBUG)
2178                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2179                                 emit_verbosecall_exit(jd);
2180 #endif /* !defined(NDEBUG) */
2181
2182 #if defined(ENABLE_THREADS)
2183                         if (checksync && code_is_synchronized(code)) {
2184                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2185
2186                                 /* we need to save the proper return value */
2187                                 switch (iptr->opc) {
2188                                 case ICMD_IRETURN:
2189                                 case ICMD_ARETURN:
2190                                 case ICMD_LRETURN:
2191                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2192                                         break;
2193                                 case ICMD_FRETURN:
2194                                 case ICMD_DRETURN:
2195                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2196                                         break;
2197                                 }
2198
2199                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2200                                 M_CALL(REG_ITMP1);
2201
2202                                 /* and now restore the proper return value */
2203                                 switch (iptr->opc) {
2204                                 case ICMD_IRETURN:
2205                                 case ICMD_ARETURN:
2206                                 case ICMD_LRETURN:
2207                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2208                                         break;
2209                                 case ICMD_FRETURN:
2210                                 case ICMD_DRETURN:
2211                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2212                                         break;
2213                                 }
2214                         }
2215 #endif
2216
2217                         /* restore saved registers */
2218
2219                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2220                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2221                         }
2222                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2223                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2224                         }
2225
2226                         /* deallocate stack */
2227
2228                         if (cd->stackframesize)
2229                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2230
2231                         /* generate method profiling code */
2232
2233                         PROFILE_CYCLE_STOP;
2234
2235                         M_RET;
2236                         }
2237                         break;
2238
2239
2240                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2241                         {
2242                                 s4 i, l;
2243                                 branch_target_t *table;
2244
2245                                 table = iptr->dst.table;
2246
2247                                 l = iptr->sx.s23.s2.tablelow;
2248                                 i = iptr->sx.s23.s3.tablehigh;
2249
2250                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2251                                 M_INTMOVE(s1, REG_ITMP1);
2252
2253                                 if (l != 0)
2254                                         M_ISUB_IMM(l, REG_ITMP1);
2255
2256                                 /* number of targets */
2257                                 i = i - l + 1;
2258
2259                 /* range check */
2260
2261                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2262                                 emit_bugt(cd, table[0].block);
2263
2264                                 /* build jump table top down and use address of lowest entry */
2265
2266                                 table += i;
2267
2268                                 while (--i >= 0) {
2269                                         dseg_add_target(cd, table->block);
2270                                         --table;
2271                                 }
2272
2273                                 /* length of dataseg after last dseg_add_target is used
2274                                    by load */
2275
2276                                 M_MOV_IMM(0, REG_ITMP2);
2277                                 dseg_adddata(cd);
2278                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2279                                 M_JMP(REG_ITMP1);
2280                         }
2281                         break;
2282
2283
2284                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2285                         {
2286                                 s4 i;
2287                                 lookup_target_t *lookup;
2288
2289                                 lookup = iptr->dst.lookup;
2290
2291                                 i = iptr->sx.s23.s2.lookupcount;
2292
2293                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2294                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2295
2296                                 while (--i >= 0) {
2297                                         M_ICMP_IMM(lookup->value, s1);
2298                                         emit_beq(cd, lookup->target.block);
2299                                         lookup++;
2300                                 }
2301
2302                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2303                                 ALIGNCODENOP;
2304                         }
2305                         break;
2306
2307
2308                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2309
2310                         REPLACEMENT_POINT_FORGC_BUILTIN(cd, iptr);
2311
2312                         bte = iptr->sx.s23.s3.bte;
2313                         md  = bte->md;
2314                         goto gen_method;
2315
2316                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2317
2318                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2319                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2320                 case ICMD_INVOKEINTERFACE:
2321
2322                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2323
2324                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2325                                 lm = NULL;
2326                                 um = iptr->sx.s23.s3.um;
2327                                 md = um->methodref->parseddesc.md;
2328                         }
2329                         else {
2330                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2331                                 um = NULL;
2332                                 md = lm->parseddesc;
2333                         }
2334
2335 gen_method:
2336                         s3 = md->paramcount;
2337
2338                         MCODECHECK((20 * s3) + 128);
2339
2340                         /* copy arguments to registers or stack location */
2341
2342                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2343                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2344                                 d   = md->params[s3].regoff;
2345
2346                                 /* already preallocated (ARGVAR)? */
2347
2348                                 if (var->flags & PREALLOC)
2349                                         continue;
2350
2351                                 if (IS_INT_LNG_TYPE(var->type)) {
2352                                         if (!md->params[s3].inmemory) {
2353                                                 s1 = emit_load(jd, iptr, var, d);
2354                                                 M_INTMOVE(s1, d);
2355                                         }
2356                                         else {
2357                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2358                                                 M_LST(s1, REG_SP, d);
2359                                         }
2360                                 }
2361                                 else {
2362                                         if (!md->params[s3].inmemory) {
2363                                                 s1 = emit_load(jd, iptr, var, d);
2364                                                 M_FLTMOVE(s1, d);
2365                                         }
2366                                         else {
2367                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2368
2369                                                 if (IS_2_WORD_TYPE(var->type))
2370                                                         M_DST(s1, REG_SP, d);
2371                                                 else
2372                                                         M_FST(s1, REG_SP, d);
2373                                         }
2374                                 }
2375                         }
2376
2377                         /* generate method profiling code */
2378
2379                         PROFILE_CYCLE_STOP;
2380
2381                         switch (iptr->opc) {
2382                         case ICMD_BUILTIN:
2383                                 if (bte->stub == NULL) {
2384                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2385                                 }
2386                                 else {
2387                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2388                                 }
2389                                 M_CALL(REG_ITMP1);
2390                                 break;
2391
2392                         case ICMD_INVOKESPECIAL:
2393                                 emit_nullpointer_check(cd, iptr, REG_A0);
2394                                 /* fall through */
2395
2396                         case ICMD_INVOKESTATIC:
2397                                 if (lm == NULL) {
2398                                         disp = dseg_add_unique_address(cd, um);
2399
2400                                         patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
2401                                                                                   um, disp);
2402                                 }
2403                                 else {
2404                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2405                                 }
2406
2407                                 M_ALD(REG_ITMP2, RIP, disp);
2408                                 M_CALL(REG_ITMP2);
2409                                 break;
2410
2411                         case ICMD_INVOKEVIRTUAL:
2412                                 if (lm == NULL) {
2413                                         patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
2414
2415                                         s1 = 0;
2416                                 }
2417                                 else {
2418                                         s1 = OFFSET(vftbl_t, table[0]) +
2419                                                 sizeof(methodptr) * lm->vftblindex;
2420                                 }
2421
2422                                 /* implicit null-pointer check */
2423                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2424                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2425                                 M_CALL(REG_ITMP3);
2426                                 break;
2427
2428                         case ICMD_INVOKEINTERFACE:
2429                                 if (lm == NULL) {
2430                                         patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
2431
2432                                         s1 = 0;
2433                                         s2 = 0;
2434                                 }
2435                                 else {
2436                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2437                                                 sizeof(methodptr) * lm->clazz->index;
2438
2439                                         s2 = sizeof(methodptr) * (lm - lm->clazz->methods);
2440                                 }
2441
2442                                 /* implicit null-pointer check */
2443                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2444                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2445                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2446                                 M_CALL(REG_ITMP3);
2447                                 break;
2448                         }
2449
2450                         /* generate method profiling code */
2451
2452                         PROFILE_CYCLE_START;
2453
2454                         /* store size of call code in replacement point */
2455
2456                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2457                         REPLACEMENT_POINT_FORGC_BUILTIN_RETURN(cd, iptr);
2458
2459                         /* store return value */
2460
2461                         switch (md->returntype.type) {
2462                         case TYPE_INT:
2463                         case TYPE_LNG:
2464                         case TYPE_ADR:
2465                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2466                                 M_INTMOVE(REG_RESULT, s1);
2467                                 emit_store_dst(jd, iptr, s1);
2468                                 break;
2469                         case TYPE_FLT:
2470                         case TYPE_DBL:
2471                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2472                                 M_FLTMOVE(REG_FRESULT, s1);
2473                                 emit_store_dst(jd, iptr, s1);
2474                                 break;
2475                         default:
2476                                 /* TYPE_VOID */
2477                                 break;
2478                         }
2479                         break;
2480
2481
2482                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2483
2484                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2485                                 /* object type cast-check */
2486
2487                                 classinfo *super;
2488                                 s4         superindex;
2489
2490                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2491                                         super      = NULL;
2492                                         superindex = 0;
2493                                 }
2494                                 else {
2495                                         super      = iptr->sx.s23.s3.c.cls;
2496                                         superindex = super->index;
2497                                 }
2498
2499                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2500
2501                                 /* if class is not resolved, check which code to call */
2502
2503                                 if (super == NULL) {
2504                                         M_TEST(s1);
2505                                         emit_label_beq(cd, BRANCH_LABEL_1);
2506
2507                                         patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2508                                                                                   iptr->sx.s23.s3.c.ref, 0);
2509
2510                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2511                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2512                                         emit_label_beq(cd, BRANCH_LABEL_2);
2513                                 }
2514
2515                                 /* interface checkcast code */
2516
2517                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2518                                         if (super != NULL) {
2519                                                 M_TEST(s1);
2520                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2521                                         }
2522
2523                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2524
2525                                         if (super == NULL) {
2526                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
2527                                                                                           iptr->sx.s23.s3.c.ref,
2528                                                                                           0);
2529                                         }
2530
2531                                         M_ILD32(REG_ITMP3,
2532                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2533                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2534                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2535
2536                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2537                                                         OFFSET(vftbl_t, interfacetable[0]) -
2538                                                         superindex * sizeof(methodptr*));
2539                                         M_TEST(REG_ITMP3);
2540                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2541
2542                                         if (super == NULL)
2543                                                 emit_label_br(cd, BRANCH_LABEL_4);
2544                                         else
2545                                                 emit_label(cd, BRANCH_LABEL_3);
2546                                 }
2547
2548                                 /* class checkcast code */
2549
2550                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2551                                         if (super == NULL) {
2552                                                 emit_label(cd, BRANCH_LABEL_2);
2553
2554                                                 cr   = iptr->sx.s23.s3.c.ref;
2555                                                 disp = dseg_add_unique_address(cd, cr);
2556
2557                                                 patcher_add_patch_ref(jd,
2558                                                                                           PATCHER_resolve_classref_to_vftbl,
2559                                                                                           cr, disp);
2560                                         }
2561                                         else {
2562                                                 M_TEST(s1);
2563                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2564
2565                                                 disp = dseg_add_address(cd, super->vftbl);
2566                                         }
2567
2568                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2569                                         M_ALD(REG_ITMP3, RIP, disp);
2570
2571                                         if (super == NULL || super->vftbl->subtype_depth >= DISPLAY_SIZE) {
2572                                                 M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_offset));
2573                                                 M_LCMP_MEMINDEX(REG_ITMP2, 0, REG_ITMP1, 0, REG_ITMP3);
2574                                                 emit_label_beq(cd, BRANCH_LABEL_6);  /* good */
2575
2576                                                 if (super == NULL) {
2577                                                         M_LCMP_IMM(OFFSET(vftbl_t, subtype_display[DISPLAY_SIZE]), REG_ITMP1);
2578                                                         emit_label_bne(cd, BRANCH_LABEL_10);  /* throw */
2579                                                 }
2580
2581                                                 M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_depth));
2582                                                 M_ICMP_MEMBASE(REG_ITMP2, OFFSET(vftbl_t, subtype_depth), REG_ITMP1);
2583                                                 emit_label_bgt(cd, BRANCH_LABEL_9);  /* throw */
2584
2585                                                 M_ALD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, subtype_overflow));
2586                                                 M_LCMP_MEMINDEX(REG_ITMP2, -8*DISPLAY_SIZE, REG_ITMP1, 3, REG_ITMP3);
2587                                                 emit_label_beq(cd, BRANCH_LABEL_7);  /* good */
2588
2589                                                 emit_label(cd, BRANCH_LABEL_9);
2590                                                 if (super == NULL)
2591                                                         emit_label(cd, BRANCH_LABEL_10);
2592
2593                                                 /* reload s1, might have been destroyed */
2594                                                 emit_load_s1(jd, iptr, REG_ITMP1);
2595                                                 M_ALD_MEM(s1, TRAP_ClassCastException);
2596
2597                                                 emit_label(cd, BRANCH_LABEL_7);
2598                                                 emit_label(cd, BRANCH_LABEL_6);
2599                                                 /* reload s1, might have been destroyed */
2600                                                 emit_load_s1(jd, iptr, REG_ITMP1);
2601                                         }
2602                                         else {
2603                                                 M_LCMP_MEMBASE(REG_ITMP2, super->vftbl->subtype_offset, REG_ITMP3);
2604                                                 emit_classcast_check(cd, iptr, BRANCH_NE, REG_ITMP3, s1);
2605                                         }
2606
2607                                         if (super != NULL)
2608                                                 emit_label(cd, BRANCH_LABEL_5);
2609                                 }
2610
2611                                 if (super == NULL) {
2612                                         emit_label(cd, BRANCH_LABEL_1);
2613                                         emit_label(cd, BRANCH_LABEL_4);
2614                                 }
2615
2616                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2617                         }
2618                         else {
2619                                 /* array type cast-check */
2620
2621                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2622                                 M_INTMOVE(s1, REG_A0);
2623
2624                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2625                                         cr   = iptr->sx.s23.s3.c.ref;
2626                                         disp = dseg_add_unique_address(cd, cr);
2627
2628                                         patcher_add_patch_ref(jd,
2629                                                                                   PATCHER_resolve_classref_to_classinfo,
2630                                                                                   cr, disp);
2631                                 }
2632                                 else {
2633                                         disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2634                                 }
2635
2636                                 M_ALD(REG_A1, RIP, disp);
2637                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2638                                 M_CALL(REG_ITMP1);
2639
2640                                 /* s1 may have been destroyed over the function call */
2641                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2642                                 M_TEST(REG_RESULT);
2643                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2644
2645                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2646                         }
2647
2648                         M_INTMOVE(s1, d);
2649                         emit_store_dst(jd, iptr, d);
2650                         break;
2651
2652                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2653
2654                         {
2655                         classinfo *super;
2656                         s4         superindex;
2657
2658                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2659                                 super      = NULL;
2660                                 superindex = 0;
2661                         }
2662                         else {
2663                                 super      = iptr->sx.s23.s3.c.cls;
2664                                 superindex = super->index;
2665                         }
2666
2667                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2668                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2669
2670                         if (s1 == d) {
2671                                 M_INTMOVE(s1, REG_ITMP1);
2672                                 s1 = REG_ITMP1;
2673                         }
2674
2675                         M_CLR(d);
2676
2677                         /* if class is not resolved, check which code to call */
2678
2679                         if (super == NULL) {
2680                                 M_TEST(s1);
2681                                 emit_label_beq(cd, BRANCH_LABEL_1);
2682
2683                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2684                                                                           iptr->sx.s23.s3.c.ref, 0);
2685
2686                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2687                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2688                                 emit_label_beq(cd, BRANCH_LABEL_2);
2689                         }
2690
2691                         /* interface instanceof code */
2692
2693                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2694                                 if (super != NULL) {
2695                                         M_TEST(s1);
2696                                         emit_label_beq(cd, BRANCH_LABEL_3);
2697                                 }
2698
2699                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2700
2701                                 if (super == NULL) {
2702                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
2703                                                                                   iptr->sx.s23.s3.c.ref, 0);
2704                                 }
2705
2706                                 M_ILD32(REG_ITMP3,
2707                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2708                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2709
2710                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2711
2712                                 M_BLE(a);
2713                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2714                                                 OFFSET(vftbl_t, interfacetable[0]) -
2715                                                 superindex * sizeof(methodptr*));
2716                                 M_TEST(REG_ITMP1);
2717                                 M_SETNE(d);
2718
2719                                 if (super == NULL)
2720                                         emit_label_br(cd, BRANCH_LABEL_4);
2721                                 else
2722                                         emit_label(cd, BRANCH_LABEL_3);
2723                         }
2724
2725                         /* class instanceof code */
2726
2727                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2728                                 if (super == NULL) {
2729                                         emit_label(cd, BRANCH_LABEL_2);
2730
2731                                         cr   = iptr->sx.s23.s3.c.ref;
2732                                         disp = dseg_add_unique_address(cd, cr);
2733
2734                                         patcher_add_patch_ref(jd,
2735                                                                                   PATCHER_resolve_classref_to_vftbl,
2736                                                                                   cr, disp);
2737                                 }
2738                                 else {
2739                                         M_TEST(s1);
2740                                         emit_label_beq(cd, BRANCH_LABEL_5);
2741
2742                                         disp = dseg_add_address(cd, super->vftbl);
2743                                 }
2744
2745                                 M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2746                                 M_ALD(REG_ITMP3, RIP, disp);
2747
2748                                 if (super == NULL || super->vftbl->subtype_depth >= DISPLAY_SIZE) {
2749                                         M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_offset));
2750                                         M_LCMP_MEMINDEX(REG_ITMP2, 0, REG_ITMP1, 0, REG_ITMP3);
2751                                         emit_label_bne(cd, BRANCH_LABEL_8); /* jump over INC/SETE */
2752                                         if (d == REG_ITMP2) {
2753                                                 M_SETE(d);
2754                                                 M_BSEXT(d, d);
2755                                         } else
2756                                                 M_LINC(d);
2757                                         emit_label_br(cd, BRANCH_LABEL_6);  /* true */
2758                                         emit_label(cd, BRANCH_LABEL_8);
2759
2760                                         if (super == NULL) {
2761                                                 M_LCMP_IMM(OFFSET(vftbl_t, subtype_display[DISPLAY_SIZE]), REG_ITMP1);
2762                                                 emit_label_bne(cd, BRANCH_LABEL_10);  /* false */
2763                                         }
2764
2765                                         M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_depth));
2766                                         M_ICMP_MEMBASE(REG_ITMP2, OFFSET(vftbl_t, subtype_depth), REG_ITMP1);
2767                                         emit_label_bgt(cd, BRANCH_LABEL_9);  /* false */
2768
2769                                         M_ALD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, subtype_overflow));
2770                                         M_LCMP_MEMINDEX(REG_ITMP2, -8*DISPLAY_SIZE, REG_ITMP1, 3, REG_ITMP3);
2771                                         M_SETE(d);
2772                                         if (d == REG_ITMP2) {
2773                                                 M_BSEXT(d, d);
2774
2775                                                 emit_label_br(cd, BRANCH_LABEL_7); /* jump over M_CLR */
2776                                         }
2777
2778                                         emit_label(cd, BRANCH_LABEL_9);
2779                                         if (super == NULL)
2780                                                 emit_label(cd, BRANCH_LABEL_10);
2781                                         if (d == REG_ITMP2) {
2782                                                 M_CLR(d);
2783
2784                                                 emit_label(cd, BRANCH_LABEL_7);
2785                                         }
2786                                         emit_label(cd, BRANCH_LABEL_6);
2787                                 }
2788                                 else {
2789                                         M_LCMP_MEMBASE(REG_ITMP2, super->vftbl->subtype_offset, REG_ITMP3);
2790                                         M_SETE(d);
2791                                         if (d == REG_ITMP2)
2792                                                 M_BSEXT(d, d);
2793                                 }
2794
2795                                 if (super != NULL)
2796                                         emit_label(cd, BRANCH_LABEL_5);
2797                         }
2798
2799                         if (super == NULL) {
2800                                 emit_label(cd, BRANCH_LABEL_1);
2801                                 emit_label(cd, BRANCH_LABEL_4);
2802                         }
2803
2804                         emit_store_dst(jd, iptr, d);
2805                         }
2806                         break;
2807
2808                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2809
2810                         /* check for negative sizes and copy sizes to stack if necessary  */
2811
2812                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2813
2814                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2815
2816                                 /* copy SAVEDVAR sizes to stack */
2817                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2818
2819                                 /* Already Preallocated? */
2820                                 if (!(var->flags & PREALLOC)) {
2821                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2822                                         M_LST(s2, REG_SP, s1 * 8);
2823                                 }
2824                         }
2825
2826                         /* a0 = dimension count */
2827
2828                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2829
2830                         /* is a patcher function set? */
2831
2832                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2833                                 cr   = iptr->sx.s23.s3.c.ref;
2834                                 disp = dseg_add_unique_address(cd, cr);
2835
2836                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
2837                                                                           cr, disp);
2838                         }
2839                         else {
2840                                 disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2841                         }
2842
2843                         /* a1 = classinfo */
2844
2845                         M_ALD(REG_A1, RIP, disp);
2846
2847                         /* a2 = pointer to dimensions = stack pointer */
2848
2849                         M_MOV(REG_SP, REG_A2);
2850
2851                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2852                         M_CALL(REG_ITMP1);
2853
2854                         /* check for exception before result assignment */
2855
2856                         emit_exception_check(cd, iptr);
2857
2858                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2859                         M_INTMOVE(REG_RESULT, s1);
2860                         emit_store_dst(jd, iptr, s1);
2861                         break;
2862
2863                 default:
2864                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2865                                                                                    iptr->opc);
2866                         return false;
2867         } /* switch */
2868
2869         } /* for instruction */
2870
2871         MCODECHECK(512); /* XXX require a lower number? */
2872
2873         /* At the end of a basic block we may have to append some nops,
2874            because the patcher stub calling code might be longer than the
2875            actual instruction. So codepatching does not change the
2876            following block unintentionally. */
2877
2878         if (cd->mcodeptr < cd->lastmcodeptr) {
2879                 while (cd->mcodeptr < cd->lastmcodeptr) {
2880                         M_NOP;
2881                 }
2882         }
2883
2884         } /* if (bptr -> flags >= BBREACHED) */
2885         } /* for basic block */
2886
2887         /* Generate patcher traps. */
2888
2889         emit_patcher_traps(jd);
2890
2891         /* everything's ok */
2892
2893         return true;
2894 }
2895
2896
2897 /* codegen_emit_stub_native ****************************************************
2898
2899    Emits a stub routine which calls a native method.
2900
2901 *******************************************************************************/
2902
2903 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f, int skipparams)
2904 {
2905         methodinfo  *m;
2906         codeinfo    *code;
2907         codegendata *cd;
2908         methoddesc  *md;
2909         int          i, j;
2910         int          s1, s2;
2911         int          disp;
2912
2913         /* Sanity check. */
2914
2915         assert(f != NULL);
2916
2917         /* Get required compiler data. */
2918
2919         m    = jd->m;
2920         code = jd->code;
2921         cd   = jd->cd;
2922
2923         /* initialize variables */
2924
2925         md = m->parseddesc;
2926
2927         /* calculate stack frame size */
2928
2929         cd->stackframesize =
2930                 sizeof(stackframeinfo_t) / SIZEOF_VOID_P +
2931                 sizeof(localref_table) / SIZEOF_VOID_P +
2932                 md->paramcount +
2933                 (md->returntype.type == TYPE_VOID ? 0 : 1) +
2934                 nmd->memuse;
2935
2936         ALIGN_ODD(cd->stackframesize);              /* keep stack 16-byte aligned */
2937
2938         /* create method header */
2939
2940         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2941         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2942         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2943         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2944         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2945
2946 #if defined(ENABLE_PROFILING)
2947         /* generate native method profiling code */
2948
2949         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
2950                 /* count frequency */
2951
2952                 M_MOV_IMM(code, REG_ITMP3);
2953                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
2954         }
2955 #endif
2956
2957         /* generate stub code */
2958
2959         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2960
2961 #if defined(ENABLE_GC_CACAO)
2962         /* Save callee saved integer registers in stackframeinfo (GC may
2963            need to recover them during a collection). */
2964
2965         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
2966                 OFFSET(stackframeinfo_t, intregs);
2967
2968         for (i = 0; i < INT_SAV_CNT; i++)
2969                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2970 #endif
2971
2972         /* save integer and float argument registers */
2973
2974         for (i = 0; i < md->paramcount; i++) {
2975                 if (!md->params[i].inmemory) {
2976                         s1 = md->params[i].regoff;
2977
2978                         switch (md->paramtypes[i].type) {
2979                         case TYPE_INT:
2980                         case TYPE_LNG:
2981                         case TYPE_ADR:
2982                                 M_LST(s1, REG_SP, i * 8);
2983                                 break;
2984                         case TYPE_FLT:
2985                         case TYPE_DBL:
2986                                 M_DST(s1, REG_SP, i * 8);
2987                                 break;
2988                         }
2989                 }
2990         }
2991
2992         /* create dynamic stack info */
2993
2994         M_MOV(REG_SP, REG_A0);
2995         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
2996         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
2997         M_CALL(REG_ITMP1);
2998
2999         /* remember class argument */
3000
3001         if (m->flags & ACC_STATIC)
3002                 M_MOV(REG_RESULT, REG_ITMP2);
3003
3004         /* restore integer and float argument registers */
3005
3006         for (i = 0; i < md->paramcount; i++) {
3007                 if (!md->params[i].inmemory) {
3008                         s1 = md->params[i].regoff;
3009
3010                         switch (md->paramtypes[i].type) {
3011                         case TYPE_INT:
3012                         case TYPE_LNG:
3013                         case TYPE_ADR:
3014                                 M_LLD(s1, REG_SP, i * 8);
3015                                 break;
3016                         case TYPE_FLT:
3017                         case TYPE_DBL:
3018                                 M_DLD(s1, REG_SP, i * 8);
3019                                 break;
3020                         }
3021                 }
3022         }
3023
3024         /* Copy or spill arguments to new locations. */
3025
3026         for (i = md->paramcount - 1, j = i + skipparams; i >= 0; i--, j--) {
3027                 s2 = nmd->params[j].regoff;
3028
3029                 switch (md->paramtypes[i].type) {
3030                 case TYPE_INT:
3031                 case TYPE_LNG:
3032                 case TYPE_ADR:
3033                         if (!md->params[i].inmemory) {
3034                                 s1 = md->params[i].regoff;
3035
3036                                 if (!nmd->params[j].inmemory)
3037                                         M_INTMOVE(s1, s2);
3038                                 else
3039                                         M_LST(s1, REG_SP, s2);
3040                         }
3041                         else {
3042                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3043                                 M_LLD(REG_ITMP1, REG_SP, s1);
3044                                 M_LST(REG_ITMP1, REG_SP, s2);
3045                         }
3046                         break;
3047                 case TYPE_FLT:
3048                         /* We only copy spilled float arguments, as the float
3049                            argument registers keep unchanged. */
3050
3051                         if (md->params[i].inmemory) {
3052                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3053
3054                                 M_FLD(REG_FTMP1, REG_SP, s1);
3055                                 M_FST(REG_FTMP1, REG_SP, s2);
3056                         }
3057                         break;
3058                 case TYPE_DBL:
3059                         if (md->params[i].inmemory) {
3060                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3061                                 M_DLD(REG_FTMP1, REG_SP, s1);
3062                                 M_DST(REG_FTMP1, REG_SP, s2);
3063                         }
3064                         break;
3065                 }
3066         }
3067
3068         /* Handle native Java methods. */
3069
3070         if (m->flags & ACC_NATIVE) {
3071                 /* put class into second argument register */
3072
3073                 if (m->flags & ACC_STATIC)
3074                         M_MOV(REG_ITMP2, REG_A1);
3075
3076                 /* put env into first argument register */
3077
3078                 M_MOV_IMM(_Jv_env, REG_A0);
3079         }
3080
3081         /* Call the native function. */
3082
3083         disp = dseg_add_functionptr(cd, f);
3084         M_ALD(REG_ITMP1, RIP, disp);
3085         M_CALL(REG_ITMP1);
3086
3087         /* save return value */
3088
3089         switch (md->returntype.type) {
3090         case TYPE_INT:
3091         case TYPE_LNG:
3092         case TYPE_ADR:
3093                 switch (md->returntype.decltype) {
3094                 case PRIMITIVETYPE_BOOLEAN:
3095                         M_BZEXT(REG_RESULT, REG_RESULT);
3096                         break;
3097                 case PRIMITIVETYPE_BYTE:
3098                         M_BSEXT(REG_RESULT, REG_RESULT);
3099                         break;
3100                 case PRIMITIVETYPE_CHAR:
3101                         M_CZEXT(REG_RESULT, REG_RESULT);
3102                         break;
3103                 case PRIMITIVETYPE_SHORT:
3104                         M_SSEXT(REG_RESULT, REG_RESULT);
3105                         break;
3106                 }
3107                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3108                 break;
3109         case TYPE_FLT:
3110         case TYPE_DBL:
3111                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3112                 break;
3113         case TYPE_VOID:
3114                 break;
3115         }
3116
3117         /* remove native stackframe info */
3118
3119         M_MOV(REG_SP, REG_A0);
3120         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3121         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3122         M_CALL(REG_ITMP1);
3123         M_MOV(REG_RESULT, REG_ITMP3);
3124
3125         /* restore return value */
3126
3127         switch (md->returntype.type) {
3128         case TYPE_INT:
3129         case TYPE_LNG:
3130         case TYPE_ADR:
3131                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3132                 break;
3133         case TYPE_FLT:
3134         case TYPE_DBL:
3135                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3136                 break;
3137         case TYPE_VOID:
3138                 break;
3139         }
3140
3141 #if defined(ENABLE_GC_CACAO)
3142         /* Restore callee saved integer registers from stackframeinfo (GC
3143            might have modified them during a collection). */
3144
3145         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
3146                 OFFSET(stackframeinfo_t, intregs);
3147
3148         for (i = 0; i < INT_SAV_CNT; i++)
3149                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3150 #endif
3151
3152         /* remove stackframe */
3153
3154         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3155
3156         /* test for exception */
3157
3158         M_TEST(REG_ITMP3);
3159         M_BNE(1);
3160         M_RET;
3161
3162         /* handle exception */
3163
3164         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3165         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3166         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3167
3168         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3169         M_JMP(REG_ITMP3);
3170 }
3171
3172
3173 /*
3174  * These are local overrides for various environment variables in Emacs.
3175  * Please do not remove this and leave it at the end of the file, where
3176  * Emacs will automagically detect them.
3177  * ---------------------------------------------------------------------
3178  * Local variables:
3179  * mode: c
3180  * indent-tabs-mode: t
3181  * c-basic-offset: 4
3182  * tab-width: 4
3183  * End:
3184  * vim:noexpandtab:sw=4:ts=4:
3185  */