src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25 */
  26
  27
  28 #include "config.h"
  29
  30 #include <assert.h>
  31 #include <stdio.h>
  32 #include <stdint.h>
  33
  34 #include "vm/types.h"
  35
  36 #include "md-abi.h"
  37
  38 #include "vm/jit/x86_64/arch.h"
  39 #include "vm/jit/x86_64/codegen.h"
  40 #include "vm/jit/x86_64/emit.h"
  41
  42 #include "mm/memory.h"
  43
  44 #include "native/jni.h"
  45 #include "native/localref.h"
  46 #include "native/native.h"
  47
  48 #include "threads/lock-common.h"
  49
  50 #include "vm/builtin.h"
  51 #include "vm/exceptions.h"
  52 #include "vm/global.h"
  53 #include "vm/primitive.h"
  54 #include "vm/stringlocal.h"
  55 #include "vm/vm.h"
  56
  57 #include "vm/jit/abi.h"
  58 #include "vm/jit/asmpart.h"
  59 #include "vm/jit/code.h"
  60 #include "vm/jit/codegen-common.h"
  61 #include "vm/jit/dseg.h"
  62 #include "vm/jit/emit-common.h"
  63 #include "vm/jit/jit.h"
  64 #include "vm/jit/linenumbertable.h"
  65 #include "vm/jit/methodheader.h"
  66 #include "vm/jit/parse.h"
  67 #include "vm/jit/patcher-common.h"
  68 #include "vm/jit/reg.h"
  69 #include "vm/jit/replace.h"
  70 #include "vm/jit/stacktrace.h"
  71 #include "vm/jit/trap.h"
  72
  73 #if defined(ENABLE_LSRA)
  74 # include "vm/jit/allocator/lsra.h"
  75 #endif
  76
  77 #include "vmcore/loader.h"
  78 #include "vmcore/options.h"
  79 #include "vmcore/statistics.h"
  80
  81
  82 /* codegen_emit ****************************************************************
  83
  84    Generates machine code.
  85
  86 *******************************************************************************/
  87
  88 bool codegen_emit(jitdata *jd)
  89 {
  90         methodinfo         *m;
  91         codeinfo           *code;
  92         codegendata        *cd;
  93         registerdata       *rd;
  94         s4                  len, s1, s2, s3, d, disp;
  95         u2                  currentline;
  96         ptrint              a;
  97         varinfo            *var, *dst;
  98         basicblock         *bptr;
  99         instruction        *iptr;
 100         constant_classref  *cr;
 101         unresolved_class   *uc;
 102         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 103         unresolved_method  *um;
 104         builtintable_entry *bte;
 105         methoddesc         *md;
 106         fieldinfo          *fi;
 107         unresolved_field   *uf;
 108         s4                  fieldtype;
 109         s4                 varindex;
 110
 111         /* get required compiler data */
 112
 113         m    = jd->m;
 114         code = jd->code;
 115         cd   = jd->cd;
 116         rd   = jd->rd;
 117
 118         /* prevent compiler warnings */
 119
 120         d   = 0;
 121         lm  = NULL;
 122         um  = NULL;
 123         bte = NULL;
 124
 125         {
 126         s4 i, p, t, l;
 127         s4 savedregs_num;
 128
 129         savedregs_num = 0;
 130
 131         /* space to save used callee saved registers */
 132
 133         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 134         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 135
 136         cd->stackframesize = rd->memuse + savedregs_num;
 137
 138 #if defined(ENABLE_THREADS)
 139         /* space to save argument of monitor_enter */
 140
 141         if (checksync && code_is_synchronized(code))
 142                 cd->stackframesize++;
 143 #endif
 144
 145         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 146            native code e.g. libc or jni (alignment problems with
 147            movaps). */
 148
 149         if (!code_is_leafmethod(code) || opt_verbosecall)
 150                 cd->stackframesize |= 0x1;
 151
 152         /* create method header */
 153
 154         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 155         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 156
 157         code->synchronizedoffset = rd->memuse * 8;
 158
 159         if (code_is_leafmethod(code))
 160                 (void) dseg_add_unique_s4(cd, 1);                  /* IsLeaf          */
 161         else
 162                 (void) dseg_add_unique_s4(cd, 0);                  /* IsLeaf          */
 163
 164         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 165         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 166
 167 #if defined(ENABLE_PROFILING)
 168         /* generate method profiling code */
 169
 170         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 171                 /* count frequency */
 172
 173                 M_MOV_IMM(code, REG_ITMP3);
 174                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 175
 176                 PROFILE_CYCLE_START;
 177         }
 178 #endif
 179
 180         /* create stack frame (if necessary) */
 181
 182         if (cd->stackframesize)
 183                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 184
 185         /* save used callee saved registers */
 186
 187         p = cd->stackframesize;
 188         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 189                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 190         }
 191         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 192                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 193         }
 194
 195         /* take arguments out of register or stack frame */
 196
 197         md = m->parseddesc;
 198
 199         for (p = 0, l = 0; p < md->paramcount; p++) {
 200                 t = md->paramtypes[p].type;
 201
 202                 varindex = jd->local_map[l * 5 + t];
 203
 204                 l++;
 205                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 206                         l++;
 207
 208                 if (varindex == UNUSED)
 209                         continue;
 210
 211                 var = VAR(varindex);
 212
 213                 s1 = md->params[p].regoff;
 214
 215                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 216                         if (!md->params[p].inmemory) {           /* register arguments    */
 217                                 if (!IS_INMEMORY(var->flags))
 218                                         M_INTMOVE(s1, var->vv.regoff);
 219                                 else
 220                                     M_LST(s1, REG_SP, var->vv.regoff);
 221                         }
 222                         else {                                 /* stack arguments       */
 223                                 if (!IS_INMEMORY(var->flags))
 224                                         /* + 8 for return address */
 225                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 226                                 else
 227                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 228                         }
 229                 }
 230                 else {                                     /* floating args         */
 231                         if (!md->params[p].inmemory) {           /* register arguments    */
 232                                 if (!IS_INMEMORY(var->flags))
 233                                         M_FLTMOVE(s1, var->vv.regoff);
 234                                 else
 235                                         M_DST(s1, REG_SP, var->vv.regoff);
 236                         }
 237                         else {                                 /* stack arguments       */
 238                                 if (!IS_INMEMORY(var->flags))
 239                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 240                                 else
 241                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 242                         }
 243                 }
 244         }
 245
 246         /* save monitorenter argument */
 247
 248 #if defined(ENABLE_THREADS)
 249         if (checksync && code_is_synchronized(code)) {
 250                 /* stack offset for monitor argument */
 251
 252                 s1 = rd->memuse;
 253
 254                 if (opt_verbosecall) {
 255                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 256
 257                         for (p = 0; p < INT_ARG_CNT; p++)
 258                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 259
 260                         for (p = 0; p < FLT_ARG_CNT; p++)
 261                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 262
 263                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 264                 }
 265
 266                 /* decide which monitor enter function to call */
 267
 268                 if (m->flags & ACC_STATIC) {
 269                         M_MOV_IMM(&m->clazz->object.header, REG_A0);
 270                 }
 271                 else {
 272                         M_TEST(REG_A0);
 273                         M_BNE(8);
 274                         M_ALD_MEM(REG_A0, TRAP_NullPointerException);
 275                 }
 276
 277                 M_AST(REG_A0, REG_SP, s1 * 8);
 278                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 279                 M_CALL(REG_ITMP1);
 280
 281                 if (opt_verbosecall) {
 282                         for (p = 0; p < INT_ARG_CNT; p++)
 283                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 284
 285                         for (p = 0; p < FLT_ARG_CNT; p++)
 286                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 287
 288                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 289                 }
 290         }
 291 #endif
 292
 293 #if !defined(NDEBUG)
 294         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 295                 emit_verbosecall_enter(jd);
 296 #endif /* !defined(NDEBUG) */
 297
 298         }
 299
 300         /* end of header generation */
 301
 302         /* create replacement points */
 303
 304         REPLACEMENT_POINTS_INIT(cd, jd);
 305
 306         /* walk through all basic blocks */
 307
 308         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 309
 310                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 311
 312                 if (bptr->flags >= BBREACHED) {
 313
 314                 /* branch resolving */
 315
 316                 codegen_resolve_branchrefs(cd, bptr);
 317
 318                 /* handle replacement points */
 319
 320                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 321
 322                 /* copy interface registers to their destination */
 323
 324                 len = bptr->indepth;
 325                 MCODECHECK(512);
 326
 327 #if defined(ENABLE_PROFILING)
 328                 /* generate basicblock profiling code */
 329
 330                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 331                         /* count frequency */
 332
 333                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 334                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 335
 336                         /* if this is an exception handler, start profiling again */
 337
 338                         if (bptr->type == BBTYPE_EXH)
 339                                 PROFILE_CYCLE_START;
 340                 }
 341 #endif
 342
 343 #if defined(ENABLE_LSRA)
 344                 if (opt_lsra) {
 345                         while (len) {
 346                                 len--;
 347                                 src = bptr->invars[len];
 348                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 349                                         if (bptr->type == BBTYPE_EXH) {
 350 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 351                                                 if (!IS_INMEMORY(src->flags))
 352                                                         d= src->vv.regoff;
 353                                                 else
 354                                                         d=REG_ITMP1;
 355                                                 M_INTMOVE(REG_ITMP1, d);
 356                                                 emit_store(jd, NULL, src, d);
 357                                         }
 358                                 }
 359                         }
 360
 361                 } else {
 362 #endif
 363
 364                 while (len) {
 365                         len--;
 366                         var = VAR(bptr->invars[len]);
 367                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 368                                 if (bptr->type == BBTYPE_EXH) {
 369                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 370                                         M_INTMOVE(REG_ITMP1, d);
 371                                         emit_store(jd, NULL, var, d);
 372                                 }
 373                         }
 374                         else {
 375                                 assert((var->flags & INOUT));
 376                         }
 377                 }
 378 #if defined(ENABLE_LSRA)
 379                 }
 380 #endif
 381                 /* walk through all instructions */
 382
 383                 len = bptr->icount;
 384                 currentline = 0;
 385
 386                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 387                         if (iptr->line != currentline) {
 388                                 linenumbertable_list_entry_add(cd, iptr->line);
 389                                 currentline = iptr->line;
 390                         }
 391
 392                         MCODECHECK(1024);                         /* 1KB should be enough */
 393
 394                 switch (iptr->opc) {
 395                 case ICMD_NOP:        /* ...  ==> ...                                 */
 396                 case ICMD_POP:        /* ..., value  ==> ...                          */
 397                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 398                         break;
 399
 400                 case ICMD_INLINE_START:
 401
 402                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 403                         break;
 404
 405                 case ICMD_INLINE_BODY:
 406
 407                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 408                         linenumbertable_list_entry_add_inline_start(cd, iptr);
 409                         linenumbertable_list_entry_add(cd, iptr->line);
 410                         break;
 411
 412                 case ICMD_INLINE_END:
 413
 414                         linenumbertable_list_entry_add_inline_end(cd, iptr);
 415                         linenumbertable_list_entry_add(cd, iptr->line);
 416                         break;
 417
 418                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 419
 420                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 421                         emit_nullpointer_check(cd, iptr, s1);
 422                         break;
 423
 424                 /* constant operations ************************************************/
 425
 426                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 427
 428                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 429                         ICONST(d, iptr->sx.val.i);
 430                         emit_store_dst(jd, iptr, d);
 431                         break;
 432
 433                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 434
 435                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 436                         LCONST(d, iptr->sx.val.l);
 437                         emit_store_dst(jd, iptr, d);
 438                         break;
 439
 440                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 441
 442                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 443                         disp = dseg_add_float(cd, iptr->sx.val.f);
 444                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 445                         emit_store_dst(jd, iptr, d);
 446                         break;
 447
 448                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 449
 450                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 451                         disp = dseg_add_double(cd, iptr->sx.val.d);
 452                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 453                         emit_store_dst(jd, iptr, d);
 454                         break;
 455
 456                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 457
 458                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 459
 460                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 461                                 cr   = iptr->sx.val.c.ref;
 462                                 disp = dseg_add_unique_address(cd, cr);
 463
 464 /*                              PROFILE_CYCLE_STOP; */
 465
 466                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
 467                                                                           cr, disp);
 468
 469 /*                              PROFILE_CYCLE_START; */
 470
 471                                 M_ALD(d, RIP, disp);
 472                         }
 473                         else {
 474                                 if (iptr->sx.val.anyptr == 0) {
 475                                         M_CLR(d);
 476                                 }
 477                                 else {
 478                                         disp = dseg_add_address(cd, iptr->sx.val.anyptr);
 479                                         M_ALD(d, RIP, disp);
 480                                 }
 481                         }
 482                         emit_store_dst(jd, iptr, d);
 483                         break;
 484
 485
 486                 /* load/store/copy/move operations ************************************/
 487
 488                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 489                 case ICMD_ALOAD:      /* s1 = local variable                          */
 490                 case ICMD_LLOAD:
 491                 case ICMD_FLOAD:
 492                 case ICMD_DLOAD:
 493                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 494                 case ICMD_LSTORE:
 495                 case ICMD_FSTORE:
 496                 case ICMD_DSTORE:
 497                 case ICMD_COPY:
 498                 case ICMD_MOVE:
 499
 500                         emit_copy(jd, iptr);
 501                         break;
 502
 503                 case ICMD_ASTORE:
 504                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 505                                 emit_copy(jd, iptr);
 506                         break;
 507
 508                 /* integer operations *************************************************/
 509
 510                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 511
 512                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 513                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 514                         M_INTMOVE(s1, d);
 515                         M_INEG(d);
 516                         emit_store_dst(jd, iptr, d);
 517                         break;
 518
 519                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 520
 521                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 522                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 523                         M_INTMOVE(s1, d);
 524                         M_LNEG(d);
 525                         emit_store_dst(jd, iptr, d);
 526                         break;
 527
 528                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 529
 530                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 531                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 532                         M_ISEXT(s1, d);
 533                         emit_store_dst(jd, iptr, d);
 534                         break;
 535
 536                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 537
 538                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 539                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 540                         M_IMOV(s1, d);
 541                         emit_store_dst(jd, iptr, d);
 542                         break;
 543
 544                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 545
 546                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 547                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 548                         M_BSEXT(s1, d);
 549                         emit_store_dst(jd, iptr, d);
 550                         break;
 551
 552                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 553
 554                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 555                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 556                         M_CZEXT(s1, d);
 557                         emit_store_dst(jd, iptr, d);
 558                         break;
 559
 560                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 561
 562                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 563                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 564                         M_SSEXT(s1, d);
 565                         emit_store_dst(jd, iptr, d);
 566                         break;
 567
 568
 569                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 570
 571                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 572                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 573                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 574                         if (s2 == d)
 575                                 M_IADD(s1, d);
 576                         else {
 577                                 M_INTMOVE(s1, d);
 578                                 M_IADD(s2, d);
 579                         }
 580                         emit_store_dst(jd, iptr, d);
 581                         break;
 582
 583                 case ICMD_IINC:
 584                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 585                                       /* sx.val.i = constant                             */
 586
 587                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 588                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 589
 590                         /* Using inc and dec is not faster than add (tested with
 591                            sieve). */
 592
 593                         M_INTMOVE(s1, d);
 594                         M_IADD_IMM(iptr->sx.val.i, d);
 595                         emit_store_dst(jd, iptr, d);
 596                         break;
 597
 598                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 599
 600                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 601                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 602                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 603                         if (s2 == d)
 604                                 M_LADD(s1, d);
 605                         else {
 606                                 M_INTMOVE(s1, d);
 607                                 M_LADD(s2, d);
 608                         }
 609                         emit_store_dst(jd, iptr, d);
 610                         break;
 611
 612                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 613                                       /* sx.val.l = constant                             */
 614
 615                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 616                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 617                         M_INTMOVE(s1, d);
 618                         if (IS_IMM32(iptr->sx.val.l))
 619                                 M_LADD_IMM(iptr->sx.val.l, d);
 620                         else {
 621                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 622                                 M_LADD(REG_ITMP2, d);
 623                         }
 624                         emit_store_dst(jd, iptr, d);
 625                         break;
 626
 627                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 628
 629                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 630                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 631                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 632                         if (s2 == d) {
 633                                 M_INTMOVE(s1, REG_ITMP1);
 634                                 M_ISUB(s2, REG_ITMP1);
 635                                 M_INTMOVE(REG_ITMP1, d);
 636                         } else {
 637                                 M_INTMOVE(s1, d);
 638                                 M_ISUB(s2, d);
 639                         }
 640                         emit_store_dst(jd, iptr, d);
 641                         break;
 642
 643                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 644                                       /* sx.val.i = constant                             */
 645
 646                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 647                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 648                         M_INTMOVE(s1, d);
 649                         M_ISUB_IMM(iptr->sx.val.i, d);
 650                         emit_store_dst(jd, iptr, d);
 651                         break;
 652
 653                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 654
 655                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 656                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 657                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 658                         if (s2 == d) {
 659                                 M_INTMOVE(s1, REG_ITMP1);
 660                                 M_LSUB(s2, REG_ITMP1);
 661                                 M_INTMOVE(REG_ITMP1, d);
 662                         } else {
 663                                 M_INTMOVE(s1, d);
 664                                 M_LSUB(s2, d);
 665                         }
 666                         emit_store_dst(jd, iptr, d);
 667                         break;
 668
 669                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 670                                       /* sx.val.l = constant                             */
 671
 672                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 673                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 674                         M_INTMOVE(s1, d);
 675                         if (IS_IMM32(iptr->sx.val.l))
 676                                 M_LSUB_IMM(iptr->sx.val.l, d);
 677                         else {
 678                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 679                                 M_LSUB(REG_ITMP2, d);
 680                         }
 681                         emit_store_dst(jd, iptr, d);
 682                         break;
 683
 684                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 685
 686                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 687                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 688                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 689                         if (s2 == d)
 690                                 M_IMUL(s1, d);
 691                         else {
 692                                 M_INTMOVE(s1, d);
 693                                 M_IMUL(s2, d);
 694                         }
 695                         emit_store_dst(jd, iptr, d);
 696                         break;
 697
 698                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 699                                       /* sx.val.i = constant                             */
 700
 701                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 702                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 703                         if (iptr->sx.val.i == 2) {
 704                                 M_INTMOVE(s1, d);
 705                                 M_ISLL_IMM(1, d);
 706                         } else
 707                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 708                         emit_store_dst(jd, iptr, d);
 709                         break;
 710
 711                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 712
 713                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 714                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 715                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 716                         if (s2 == d)
 717                                 M_LMUL(s1, d);
 718                         else {
 719                                 M_INTMOVE(s1, d);
 720                                 M_LMUL(s2, d);
 721                         }
 722                         emit_store_dst(jd, iptr, d);
 723                         break;
 724
 725                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 726                                       /* sx.val.l = constant                             */
 727
 728                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 729                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 730                         if (IS_IMM32(iptr->sx.val.l))
 731                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 732                         else {
 733                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 734                                 M_INTMOVE(s1, d);
 735                                 M_LMUL(REG_ITMP2, d);
 736                         }
 737                         emit_store_dst(jd, iptr, d);
 738                         break;
 739
 740                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 741
 742                         s1 = emit_load_s1(jd, iptr, RAX);
 743                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 744                         d = codegen_reg_of_dst(jd, iptr, RAX);
 745
 746                         M_INTMOVE(s1, RAX);
 747                         M_INTMOVE(s2, REG_ITMP3);
 748                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 749
 750                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 751
 752                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 753                         M_BNE(4 + 6);
 754                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 755                         M_BEQ(1 + 3);                                          /* 6 bytes */
 756
 757                         emit_cltd(cd);                                         /* 1 byte  */
 758                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 759
 760                         M_INTMOVE(RAX, d);
 761                         emit_store_dst(jd, iptr, d);
 762                         dst = VAROP(iptr->dst);
 763                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 764                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 765                         break;
 766
 767                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 768
 769                         s1 = emit_load_s1(jd, iptr, RAX);
 770                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 771                         d = codegen_reg_of_dst(jd, iptr, RDX);
 772
 773                         M_INTMOVE(s1, RAX);
 774                         M_INTMOVE(s2, REG_ITMP3);
 775                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 776
 777                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 778
 779                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 780                         M_BNE(3 + 4 + 6);
 781                         M_CLR(RDX);                                            /* 3 bytes */
 782                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 783                         M_BEQ(1 + 3);                                          /* 6 bytes */
 784
 785                         emit_cltd(cd);                                         /* 1 byte  */
 786                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 787
 788                         M_INTMOVE(RDX, d);
 789                         emit_store_dst(jd, iptr, d);
 790                         dst = VAROP(iptr->dst);
 791                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 792                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 793                         break;
 794
 795                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 796                                       /* sx.val.i = constant                             */
 797
 798                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 799                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 800                         M_INTMOVE(s1, REG_ITMP1);
 801                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 802                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 803                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 804                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 805                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 806                         emit_store_dst(jd, iptr, d);
 807                         break;
 808
 809                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 810                                       /* sx.val.i = constant                             */
 811
 812                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 813                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 814                         M_INTMOVE(s1, REG_ITMP1);
 815                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 816                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 817                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 818                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 819                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 820                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 821                         emit_store_dst(jd, iptr, d);
 822                         break;
 823
 824
 825                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 826
 827                         s1 = emit_load_s1(jd, iptr, RAX);
 828                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 829                         d = codegen_reg_of_dst(jd, iptr, RAX);
 830
 831                         M_INTMOVE(s1, RAX);
 832                         M_INTMOVE(s2, REG_ITMP3);
 833                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 834
 835                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 836
 837                         /* check as described in jvm spec */
 838                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 839                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 840                         M_BNE(4 + 6);
 841                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 842                         M_BEQ(2 + 3);                                          /* 6 bytes */
 843
 844                         emit_cqto(cd);                                         /* 2 bytes */
 845                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 846
 847                         M_INTMOVE(RAX, d);
 848                         emit_store_dst(jd, iptr, d);
 849                         dst = VAROP(iptr->dst);
 850                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 851                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 852                         break;
 853
 854                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 855
 856                         s1 = emit_load_s1(jd, iptr, RAX);
 857                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 858                         d = codegen_reg_of_dst(jd, iptr, RDX);
 859
 860                         M_INTMOVE(s1, RAX);
 861                         M_INTMOVE(s2, REG_ITMP3);
 862                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 863
 864                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 865
 866                         /* check as described in jvm spec */
 867                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 868                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 869                         M_BNE(3 + 4 + 6);
 870                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 871                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 872                         M_BEQ(2 + 3);                                          /* 6 bytes */
 873
 874                         emit_cqto(cd);                                         /* 2 bytes */
 875                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 876
 877                         M_INTMOVE(RDX, d);
 878                         emit_store_dst(jd, iptr, d);
 879                         dst = VAROP(iptr->dst);
 880                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 881                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 882                         break;
 883
 884                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 885                                       /* sx.val.i = constant                             */
 886
 887                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 888                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 889                         M_INTMOVE(s1, REG_ITMP1);
 890                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 891                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 892                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 893                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 894                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 895                         emit_store_dst(jd, iptr, d);
 896                         break;
 897
 898                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 899                                       /* sx.val.l = constant                             */
 900
 901                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 902                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 903                         M_INTMOVE(s1, REG_ITMP1);
 904                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 905                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 906                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 907                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 908                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 909                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 910                         emit_store_dst(jd, iptr, d);
 911                         break;
 912
 913                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 914
 915                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 916                         emit_ishift(jd, SHIFT_SHL, iptr);
 917                         break;
 918
 919                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 920                                       /* sx.val.i = constant                             */
 921
 922                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 923                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 924                         M_INTMOVE(s1, d);
 925                         M_ISLL_IMM(iptr->sx.val.i, d);
 926                         emit_store_dst(jd, iptr, d);
 927                         break;
 928
 929                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 930
 931                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 932                         emit_ishift(jd, SHIFT_SAR, iptr);
 933                         break;
 934
 935                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 936                                       /* sx.val.i = constant                             */
 937
 938                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 939                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 940                         M_INTMOVE(s1, d);
 941                         M_ISRA_IMM(iptr->sx.val.i, d);
 942                         emit_store_dst(jd, iptr, d);
 943                         break;
 944
 945                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 946
 947                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 948                         emit_ishift(jd, SHIFT_SHR, iptr);
 949                         break;
 950
 951                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 952                                       /* sx.val.i = constant                             */
 953
 954                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 955                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 956                         M_INTMOVE(s1, d);
 957                         M_ISRL_IMM(iptr->sx.val.i, d);
 958                         emit_store_dst(jd, iptr, d);
 959                         break;
 960
 961                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 962
 963                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 964                         emit_lshift(jd, SHIFT_SHL, iptr);
 965                         break;
 966
 967         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 968                                           /* sx.val.i = constant                             */
 969
 970                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 971                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 972                         M_INTMOVE(s1, d);
 973                         M_LSLL_IMM(iptr->sx.val.i, d);
 974                         emit_store_dst(jd, iptr, d);
 975                         break;
 976
 977                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 978
 979                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 980                         emit_lshift(jd, SHIFT_SAR, iptr);
 981                         break;
 982
 983                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 984                                       /* sx.val.i = constant                             */
 985
 986                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 987                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 988                         M_INTMOVE(s1, d);
 989                         M_LSRA_IMM(iptr->sx.val.i, d);
 990                         emit_store_dst(jd, iptr, d);
 991                         break;
 992
 993                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 994
 995                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 996                         emit_lshift(jd, SHIFT_SHR, iptr);
 997                         break;
 998
 999                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1000                                       /* sx.val.l = constant                             */
1001
1002                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1003                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1004                         M_INTMOVE(s1, d);
1005                         M_LSRL_IMM(iptr->sx.val.i, d);
1006                         emit_store_dst(jd, iptr, d);
1007                         break;
1008
1009                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1010
1011                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1012                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1013                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1014                         if (s2 == d)
1015                                 M_IAND(s1, d);
1016                         else {
1017                                 M_INTMOVE(s1, d);
1018                                 M_IAND(s2, d);
1019                         }
1020                         emit_store_dst(jd, iptr, d);
1021                         break;
1022
1023                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1024                                       /* sx.val.i = constant                             */
1025
1026                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1027                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1028                         M_INTMOVE(s1, d);
1029                         M_IAND_IMM(iptr->sx.val.i, d);
1030                         emit_store_dst(jd, iptr, d);
1031                         break;
1032
1033                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1034
1035                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1036                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1037                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1038                         if (s2 == d)
1039                                 M_LAND(s1, d);
1040                         else {
1041                                 M_INTMOVE(s1, d);
1042                                 M_LAND(s2, d);
1043                         }
1044                         emit_store_dst(jd, iptr, d);
1045                         break;
1046
1047                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1048                                       /* sx.val.l = constant                             */
1049
1050                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1051                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1052                         M_INTMOVE(s1, d);
1053                         if (IS_IMM32(iptr->sx.val.l))
1054                                 M_LAND_IMM(iptr->sx.val.l, d);
1055                         else {
1056                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1057                                 M_LAND(REG_ITMP2, d);
1058                         }
1059                         emit_store_dst(jd, iptr, d);
1060                         break;
1061
1062                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1063
1064                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1065                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1066                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1067                         if (s2 == d)
1068                                 M_IOR(s1, d);
1069                         else {
1070                                 M_INTMOVE(s1, d);
1071                                 M_IOR(s2, d);
1072                         }
1073                         emit_store_dst(jd, iptr, d);
1074                         break;
1075
1076                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1077                                       /* sx.val.i = constant                             */
1078
1079                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1080                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1081                         M_INTMOVE(s1, d);
1082                         M_IOR_IMM(iptr->sx.val.i, d);
1083                         emit_store_dst(jd, iptr, d);
1084                         break;
1085
1086                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1087
1088                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1089                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1090                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1091                         if (s2 == d)
1092                                 M_LOR(s1, d);
1093                         else {
1094                                 M_INTMOVE(s1, d);
1095                                 M_LOR(s2, d);
1096                         }
1097                         emit_store_dst(jd, iptr, d);
1098                         break;
1099
1100                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1101                                       /* sx.val.l = constant                             */
1102
1103                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1104                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1105                         M_INTMOVE(s1, d);
1106                         if (IS_IMM32(iptr->sx.val.l))
1107                                 M_LOR_IMM(iptr->sx.val.l, d);
1108                         else {
1109                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1110                                 M_LOR(REG_ITMP2, d);
1111                         }
1112                         emit_store_dst(jd, iptr, d);
1113                         break;
1114
1115                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1116
1117                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1118                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1119                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1120                         if (s2 == d)
1121                                 M_IXOR(s1, d);
1122                         else {
1123                                 M_INTMOVE(s1, d);
1124                                 M_IXOR(s2, d);
1125                         }
1126                         emit_store_dst(jd, iptr, d);
1127                         break;
1128
1129                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1130                                       /* sx.val.i = constant                             */
1131
1132                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1133                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1134                         M_INTMOVE(s1, d);
1135                         M_IXOR_IMM(iptr->sx.val.i, d);
1136                         emit_store_dst(jd, iptr, d);
1137                         break;
1138
1139                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1140
1141                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1142                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1143                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1144                         if (s2 == d)
1145                                 M_LXOR(s1, d);
1146                         else {
1147                                 M_INTMOVE(s1, d);
1148                                 M_LXOR(s2, d);
1149                         }
1150                         emit_store_dst(jd, iptr, d);
1151                         break;
1152
1153                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1154                                       /* sx.val.l = constant                             */
1155
1156                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1157                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1158                         M_INTMOVE(s1, d);
1159                         if (IS_IMM32(iptr->sx.val.l))
1160                                 M_LXOR_IMM(iptr->sx.val.l, d);
1161                         else {
1162                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1163                                 M_LXOR(REG_ITMP2, d);
1164                         }
1165                         emit_store_dst(jd, iptr, d);
1166                         break;
1167
1168
1169                 /* floating operations ************************************************/
1170
1171                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1172
1173                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1174                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1175                         disp = dseg_add_s4(cd, 0x80000000);
1176                         M_FLTMOVE(s1, d);
1177                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1178                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1179                         emit_store_dst(jd, iptr, d);
1180                         break;
1181
1182                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1183
1184                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1185                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1186                         disp = dseg_add_s8(cd, 0x8000000000000000);
1187                         M_FLTMOVE(s1, d);
1188                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1189                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1190                         emit_store_dst(jd, iptr, d);
1191                         break;
1192
1193                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1194
1195                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1196                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1197                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1198                         if (s2 == d)
1199                                 M_FADD(s1, d);
1200                         else {
1201                                 M_FLTMOVE(s1, d);
1202                                 M_FADD(s2, d);
1203                         }
1204                         emit_store_dst(jd, iptr, d);
1205                         break;
1206
1207                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1208
1209                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1210                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1211                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1212                         if (s2 == d)
1213                                 M_DADD(s1, d);
1214                         else {
1215                                 M_FLTMOVE(s1, d);
1216                                 M_DADD(s2, d);
1217                         }
1218                         emit_store_dst(jd, iptr, d);
1219                         break;
1220
1221                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1222
1223                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1224                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1225                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1226                         if (s2 == d) {
1227                                 M_FLTMOVE(s2, REG_FTMP2);
1228                                 s2 = REG_FTMP2;
1229                         }
1230                         M_FLTMOVE(s1, d);
1231                         M_FSUB(s2, d);
1232                         emit_store_dst(jd, iptr, d);
1233                         break;
1234
1235                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1236
1237                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1238                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1239                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1240                         if (s2 == d) {
1241                                 M_FLTMOVE(s2, REG_FTMP2);
1242                                 s2 = REG_FTMP2;
1243                         }
1244                         M_FLTMOVE(s1, d);
1245                         M_DSUB(s2, d);
1246                         emit_store_dst(jd, iptr, d);
1247                         break;
1248
1249                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1250
1251                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1252                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1253                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1254                         if (s2 == d)
1255                                 M_FMUL(s1, d);
1256                         else {
1257                                 M_FLTMOVE(s1, d);
1258                                 M_FMUL(s2, d);
1259                         }
1260                         emit_store_dst(jd, iptr, d);
1261                         break;
1262
1263                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1264
1265                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1266                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1267                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1268                         if (s2 == d)
1269                                 M_DMUL(s1, d);
1270                         else {
1271                                 M_FLTMOVE(s1, d);
1272                                 M_DMUL(s2, d);
1273                         }
1274                         emit_store_dst(jd, iptr, d);
1275                         break;
1276
1277                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1278
1279                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1280                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1281                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1282                         if (s2 == d) {
1283                                 M_FLTMOVE(s2, REG_FTMP2);
1284                                 s2 = REG_FTMP2;
1285                         }
1286                         M_FLTMOVE(s1, d);
1287                         M_FDIV(s2, d);
1288                         emit_store_dst(jd, iptr, d);
1289                         break;
1290
1291                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1292
1293                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1294                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1295                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1296                         if (s2 == d) {
1297                                 M_FLTMOVE(s2, REG_FTMP2);
1298                                 s2 = REG_FTMP2;
1299                         }
1300                         M_FLTMOVE(s1, d);
1301                         M_DDIV(s2, d);
1302                         emit_store_dst(jd, iptr, d);
1303                         break;
1304
1305                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1306
1307                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1308                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1309                         M_CVTIF(s1, d);
1310                         emit_store_dst(jd, iptr, d);
1311                         break;
1312
1313                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1314
1315                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1316                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1317                         M_CVTID(s1, d);
1318                         emit_store_dst(jd, iptr, d);
1319                         break;
1320
1321                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1322
1323                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1324                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1325                         M_CVTLF(s1, d);
1326                         emit_store_dst(jd, iptr, d);
1327                         break;
1328
1329                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1330
1331                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1332                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1333                         M_CVTLD(s1, d);
1334                         emit_store_dst(jd, iptr, d);
1335                         break;
1336
1337                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1338
1339                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1340                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1341                         M_CVTFI(s1, d);
1342                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1343                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1344                                 ((REG_RESULT == d) ? 0 : 3);
1345                         M_BNE(disp);
1346                         M_FLTMOVE(s1, REG_FTMP1);
1347                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1348                         M_CALL(REG_ITMP2);
1349                         M_INTMOVE(REG_RESULT, d);
1350                         emit_store_dst(jd, iptr, d);
1351                         break;
1352
1353                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1354
1355                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1356                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1357                         M_CVTDI(s1, d);
1358                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1359                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1360                                 ((REG_RESULT == d) ? 0 : 3);
1361                         M_BNE(disp);
1362                         M_FLTMOVE(s1, REG_FTMP1);
1363                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1364                         M_CALL(REG_ITMP2);
1365                         M_INTMOVE(REG_RESULT, d);
1366                         emit_store_dst(jd, iptr, d);
1367                         break;
1368
1369                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1370
1371                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1372                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1373                         M_CVTFL(s1, d);
1374                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1375                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1376                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1377                                 ((REG_RESULT == d) ? 0 : 3);
1378                         M_BNE(disp);
1379                         M_FLTMOVE(s1, REG_FTMP1);
1380                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1381                         M_CALL(REG_ITMP2);
1382                         M_INTMOVE(REG_RESULT, d);
1383                         emit_store_dst(jd, iptr, d);
1384                         break;
1385
1386                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1387
1388                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1389                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1390                         M_CVTDL(s1, d);
1391                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1392                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1393                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1394                                 ((REG_RESULT == d) ? 0 : 3);
1395                         M_BNE(disp);
1396                         M_FLTMOVE(s1, REG_FTMP1);
1397                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1398                         M_CALL(REG_ITMP2);
1399                         M_INTMOVE(REG_RESULT, d);
1400                         emit_store_dst(jd, iptr, d);
1401                         break;
1402
1403                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1404
1405                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1406                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1407                         M_CVTFD(s1, d);
1408                         emit_store_dst(jd, iptr, d);
1409                         break;
1410
1411                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1412
1413                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1414                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1415                         M_CVTDF(s1, d);
1416                         emit_store_dst(jd, iptr, d);
1417                         break;
1418
1419                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1420                                           /* == => 0, < => 1, > => -1 */
1421
1422                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1423                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1424                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1425                         M_CLR(d);
1426                         M_MOV_IMM(1, REG_ITMP1);
1427                         M_MOV_IMM(-1, REG_ITMP2);
1428                         emit_ucomiss_reg_reg(cd, s1, s2);
1429                         M_CMOVULT(REG_ITMP1, d);
1430                         M_CMOVUGT(REG_ITMP2, d);
1431                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1432                         emit_store_dst(jd, iptr, d);
1433                         break;
1434
1435                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1436                                           /* == => 0, < => 1, > => -1 */
1437
1438                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1439                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1440                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1441                         M_CLR(d);
1442                         M_MOV_IMM(1, REG_ITMP1);
1443                         M_MOV_IMM(-1, REG_ITMP2);
1444                         emit_ucomiss_reg_reg(cd, s1, s2);
1445                         M_CMOVULT(REG_ITMP1, d);
1446                         M_CMOVUGT(REG_ITMP2, d);
1447                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1448                         emit_store_dst(jd, iptr, d);
1449                         break;
1450
1451                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1452                                           /* == => 0, < => 1, > => -1 */
1453
1454                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1455                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1456                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1457                         M_CLR(d);
1458                         M_MOV_IMM(1, REG_ITMP1);
1459                         M_MOV_IMM(-1, REG_ITMP2);
1460                         emit_ucomisd_reg_reg(cd, s1, s2);
1461                         M_CMOVULT(REG_ITMP1, d);
1462                         M_CMOVUGT(REG_ITMP2, d);
1463                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1464                         emit_store_dst(jd, iptr, d);
1465                         break;
1466
1467                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1468                                           /* == => 0, < => 1, > => -1 */
1469
1470                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1471                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1472                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1473                         M_CLR(d);
1474                         M_MOV_IMM(1, REG_ITMP1);
1475                         M_MOV_IMM(-1, REG_ITMP2);
1476                         emit_ucomisd_reg_reg(cd, s1, s2);
1477                         M_CMOVULT(REG_ITMP1, d);
1478                         M_CMOVUGT(REG_ITMP2, d);
1479                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1480                         emit_store_dst(jd, iptr, d);
1481                         break;
1482
1483
1484                 /* memory operations **************************************************/
1485
1486                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1487
1488                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1489                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1490                         /* implicit null-pointer check */
1491                         M_ILD(d, s1, OFFSET(java_array_t, size));
1492                         emit_store_dst(jd, iptr, d);
1493                         break;
1494
1495                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1496
1497                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1498                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1499                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1500                         /* implicit null-pointer check */
1501                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1502                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1503                         emit_store_dst(jd, iptr, d);
1504                         break;
1505
1506                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1507
1508                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1509                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1510                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1511                         /* implicit null-pointer check */
1512                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1513                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1514                         emit_store_dst(jd, iptr, d);
1515                         break;
1516
1517                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1518
1519                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1520                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1521                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1522                         /* implicit null-pointer check */
1523                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1524                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1525                         emit_store_dst(jd, iptr, d);
1526                         break;
1527
1528                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1529
1530                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1531                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1532                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1533                         /* implicit null-pointer check */
1534                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1535                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1536                         emit_store_dst(jd, iptr, d);
1537                         break;
1538
1539                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1540
1541                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1542                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1543                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1544                         /* implicit null-pointer check */
1545                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1546                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1547                         emit_store_dst(jd, iptr, d);
1548                         break;
1549
1550                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1551
1552                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1553                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1554                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1555                         /* implicit null-pointer check */
1556                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1557                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1558                         emit_store_dst(jd, iptr, d);
1559                         break;
1560
1561                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1562
1563                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1564                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1565                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1566                         /* implicit null-pointer check */
1567                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1568                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1569                         emit_store_dst(jd, iptr, d);
1570                         break;
1571
1572                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1573
1574                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1575                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1576                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1577                         /* implicit null-pointer check */
1578                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1579                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1580                         emit_store_dst(jd, iptr, d);
1581                         break;
1582
1583
1584                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1585
1586                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1587                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1588                         /* implicit null-pointer check */
1589                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1590                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1591                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1592                         break;
1593
1594                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1595
1596                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1597                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1598                         /* implicit null-pointer check */
1599                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1600                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1601                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1602                         break;
1603
1604                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1605
1606                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1607                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1608                         /* implicit null-pointer check */
1609                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1610                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1611                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1612                         break;
1613
1614                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1615
1616                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1617                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1618                         /* implicit null-pointer check */
1619                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1620                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1621                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1622                         break;
1623
1624                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1625
1626                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1627                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1628                         /* implicit null-pointer check */
1629                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1630                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1631                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1632                         break;
1633
1634                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1635
1636                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1637                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1638                         /* implicit null-pointer check */
1639                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1640                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1641                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1642                         break;
1643
1644                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1645
1646                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1647                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1648                         /* implicit null-pointer check */
1649                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1650                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1651                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1652                         break;
1653
1654                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1655
1656                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1657                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1658                         /* implicit null-pointer check */
1659                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1660                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1661
1662                         M_MOV(s1, REG_A0);
1663                         M_MOV(s3, REG_A1);
1664                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
1665                         M_CALL(REG_ITMP1);
1666                         emit_arraystore_check(cd, iptr);
1667
1668                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1669                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1670                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1671                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1672                         break;
1673
1674
1675                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1676
1677                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1678                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1679                         /* implicit null-pointer check */
1680                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1681                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1682                         break;
1683
1684                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1685
1686                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1687                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1688                         /* implicit null-pointer check */
1689                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1690                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1691                         break;
1692
1693                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1694
1695                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1696                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1697                         /* implicit null-pointer check */
1698                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1699                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1700                         break;
1701
1702                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1703
1704                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1705                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1706                         /* implicit null-pointer check */
1707                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1708                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1709                         break;
1710
1711                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1712
1713                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1714                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1715                         /* implicit null-pointer check */
1716                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1717
1718                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1719                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1720                         }
1721                         else {
1722                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1723                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1724                         }
1725                         break;
1726
1727                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1728
1729                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1730                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1731                         /* implicit null-pointer check */
1732                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1733                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1734                         break;
1735
1736
1737                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1738
1739                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1740                                 uf        = iptr->sx.s23.s3.uf;
1741                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1742                                 disp      = dseg_add_unique_address(cd, uf);
1743
1744 /*                              PROFILE_CYCLE_STOP; */
1745
1746                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1747
1748 /*                              PROFILE_CYCLE_START; */
1749                         }
1750                         else {
1751                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1752                                 fieldtype = fi->type;
1753                                 disp      = dseg_add_address(cd, fi->value);
1754
1755                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1756                                         PROFILE_CYCLE_STOP;
1757
1758                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1759                                                                                   fi->clazz, 0);
1760
1761                                         PROFILE_CYCLE_START;
1762                                 }
1763                         }
1764
1765                         /* This approach is much faster than moving the field
1766                            address inline into a register. */
1767
1768                         M_ALD(REG_ITMP1, RIP, disp);
1769
1770                         switch (fieldtype) {
1771                         case TYPE_INT:
1772                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1773                                 M_ILD(d, REG_ITMP1, 0);
1774                                 break;
1775                         case TYPE_LNG:
1776                         case TYPE_ADR:
1777                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1778                                 M_LLD(d, REG_ITMP1, 0);
1779                                 break;
1780                         case TYPE_FLT:
1781                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1782                                 M_FLD(d, REG_ITMP1, 0);
1783                                 break;
1784                         case TYPE_DBL:
1785                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1786                                 M_DLD(d, REG_ITMP1, 0);
1787                                 break;
1788                         }
1789                         emit_store_dst(jd, iptr, d);
1790                         break;
1791
1792                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1793
1794                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1795                                 uf        = iptr->sx.s23.s3.uf;
1796                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1797                                 disp      = dseg_add_unique_address(cd, uf);
1798
1799 /*                              PROFILE_CYCLE_STOP; */
1800
1801                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1802
1803 /*                              PROFILE_CYCLE_START; */
1804                         }
1805                         else {
1806                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1807                                 fieldtype = fi->type;
1808                                 disp      = dseg_add_address(cd, fi->value);
1809
1810                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1811                                         PROFILE_CYCLE_STOP;
1812
1813                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1814                                                                                   fi->clazz, 0);
1815
1816                                         PROFILE_CYCLE_START;
1817                                 }
1818                         }
1819
1820                         /* This approach is much faster than moving the field
1821                            address inline into a register. */
1822
1823                         M_ALD(REG_ITMP1, RIP, disp);
1824
1825                         switch (fieldtype) {
1826                         case TYPE_INT:
1827                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1828                                 M_IST(s1, REG_ITMP1, 0);
1829                                 break;
1830                         case TYPE_LNG:
1831                         case TYPE_ADR:
1832                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1833                                 M_LST(s1, REG_ITMP1, 0);
1834                                 break;
1835                         case TYPE_FLT:
1836                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1837                                 M_FST(s1, REG_ITMP1, 0);
1838                                 break;
1839                         case TYPE_DBL:
1840                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1841                                 M_DST(s1, REG_ITMP1, 0);
1842                                 break;
1843                         }
1844                         break;
1845
1846                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1847                                           /* val = value (in current instruction)     */
1848                                           /* following NOP)                           */
1849
1850                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1851                                 uf        = iptr->sx.s23.s3.uf;
1852                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1853                                 disp      = dseg_add_unique_address(cd, uf);
1854
1855 /*                              PROFILE_CYCLE_STOP; */
1856
1857                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1858
1859 /*                              PROFILE_CYCLE_START; */
1860                         }
1861                         else {
1862                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1863                                 fieldtype = fi->type;
1864                                 disp      = dseg_add_address(cd, fi->value);
1865
1866                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1867                                         PROFILE_CYCLE_STOP;
1868
1869                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1870                                                                                   fi->clazz, 0);
1871
1872                                         PROFILE_CYCLE_START;
1873                                 }
1874                         }
1875
1876                         /* This approach is much faster than moving the field
1877                            address inline into a register. */
1878
1879                         M_ALD(REG_ITMP1, RIP, disp);
1880
1881                         switch (fieldtype) {
1882                         case TYPE_INT:
1883                         case TYPE_FLT:
1884                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1885                                 break;
1886                         case TYPE_LNG:
1887                         case TYPE_ADR:
1888                         case TYPE_DBL:
1889                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1890                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1891                                 else {
1892                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1893                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1894                                 }
1895                                 break;
1896                         }
1897                         break;
1898
1899                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1900
1901                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1902
1903                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1904                                 uf        = iptr->sx.s23.s3.uf;
1905                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1906                                 disp      = 0;
1907
1908 /*                              PROFILE_CYCLE_STOP; */
1909
1910                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1911
1912 /*                              PROFILE_CYCLE_START; */
1913                         }
1914                         else {
1915                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1916                                 fieldtype = fi->type;
1917                                 disp      = fi->offset;
1918                         }
1919
1920                         /* implicit null-pointer check */
1921                         switch (fieldtype) {
1922                         case TYPE_INT:
1923                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1924                                 M_ILD32(d, s1, disp);
1925                                 break;
1926                         case TYPE_LNG:
1927                         case TYPE_ADR:
1928                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1929                                 M_LLD32(d, s1, disp);
1930                                 break;
1931                         case TYPE_FLT:
1932                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1933                                 M_FLD32(d, s1, disp);
1934                                 break;
1935                         case TYPE_DBL:
1936                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1937                                 M_DLD32(d, s1, disp);
1938                                 break;
1939                         }
1940                         emit_store_dst(jd, iptr, d);
1941                         break;
1942
1943                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1944
1945                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1946                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1947
1948                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1949                                 uf        = iptr->sx.s23.s3.uf;
1950                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1951                                 disp      = 0;
1952
1953 /*                              PROFILE_CYCLE_STOP; */
1954
1955                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1956
1957 /*                              PROFILE_CYCLE_START; */
1958                         }
1959                         else {
1960                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1961                                 fieldtype = fi->type;
1962                                 disp      = fi->offset;
1963                         }
1964
1965                         /* implicit null-pointer check */
1966                         switch (fieldtype) {
1967                         case TYPE_INT:
1968                                 M_IST32(s2, s1, disp);
1969                                 break;
1970                         case TYPE_LNG:
1971                         case TYPE_ADR:
1972                                 M_LST32(s2, s1, disp);
1973                                 break;
1974                         case TYPE_FLT:
1975                                 M_FST32(s2, s1, disp);
1976                                 break;
1977                         case TYPE_DBL:
1978                                 M_DST32(s2, s1, disp);
1979                                 break;
1980                         }
1981                         break;
1982
1983                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
1984                                           /* val = value (in current instruction)     */
1985                                           /* following NOP)                           */
1986
1987                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1988
1989                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1990                                 uf        = iptr->sx.s23.s3.uf;
1991                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1992                                 disp      = 0;
1993
1994 /*                              PROFILE_CYCLE_STOP; */
1995
1996                                 patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
1997
1998 /*                              PROFILE_CYCLE_START; */
1999                         }
2000                         else {
2001                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2002                                 fieldtype = fi->type;
2003                                 disp      = fi->offset;
2004                         }
2005
2006                         /* implicit null-pointer check */
2007                         switch (fieldtype) {
2008                         case TYPE_INT:
2009                         case TYPE_FLT:
2010                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2011                                 break;
2012                         case TYPE_LNG:
2013                         case TYPE_ADR:
2014                         case TYPE_DBL:
2015                                 /* XXX why no check for IS_IMM32? */
2016                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2017                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2018                                 break;
2019                         }
2020                         break;
2021
2022
2023                 /* branch operations **************************************************/
2024
2025                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2026
2027                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2028                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2029
2030                         PROFILE_CYCLE_STOP;
2031
2032 #ifdef ENABLE_VERIFIER
2033                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2034                                 uc = iptr->sx.s23.s2.uc;
2035
2036                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2037                         }
2038 #endif /* ENABLE_VERIFIER */
2039
2040                         M_CALL_IMM(0);                            /* passing exception pc */
2041                         M_POP(REG_ITMP2_XPC);
2042
2043                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2044                         M_JMP(REG_ITMP3);
2045                         break;
2046
2047                 case ICMD_GOTO:         /* ... ==> ...                                */
2048                 case ICMD_RET:
2049
2050                         emit_br(cd, iptr->dst.block);
2051                         ALIGNCODENOP;
2052                         break;
2053
2054                 case ICMD_JSR:          /* ... ==> ...                                */
2055
2056                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2057                         ALIGNCODENOP;
2058                         break;
2059
2060                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2061                 case ICMD_IFNONNULL:
2062
2063                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2064                         M_TEST(s1);
2065                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2066                         break;
2067
2068                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2069                 case ICMD_IFLT:
2070                 case ICMD_IFLE:
2071                 case ICMD_IFNE:
2072                 case ICMD_IFGT:
2073                 case ICMD_IFGE:
2074
2075                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2076                         M_ICMP_IMM(iptr->sx.val.i, s1);
2077                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2078                         break;
2079
2080                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2081                 case ICMD_IF_LNE:
2082                 case ICMD_IF_LLT:
2083                 case ICMD_IF_LGE:
2084                 case ICMD_IF_LGT:
2085                 case ICMD_IF_LLE:
2086
2087                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2088                         if (IS_IMM32(iptr->sx.val.l))
2089                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2090                         else {
2091                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2092                                 M_LCMP(REG_ITMP2, s1);
2093                         }
2094                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2095                         break;
2096
2097                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2098                 case ICMD_IF_ICMPNE:
2099                 case ICMD_IF_ICMPLT:
2100                 case ICMD_IF_ICMPGE:
2101                 case ICMD_IF_ICMPGT:
2102                 case ICMD_IF_ICMPLE:
2103
2104                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2105                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2106                         M_ICMP(s2, s1);
2107                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2108                         break;
2109
2110                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2111                 case ICMD_IF_ACMPNE:
2112
2113                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2114                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2115                         M_LCMP(s2, s1);
2116                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2117                         break;
2118
2119                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2120                 case ICMD_IF_LCMPNE:
2121                 case ICMD_IF_LCMPLT:
2122                 case ICMD_IF_LCMPGE:
2123                 case ICMD_IF_LCMPGT:
2124                 case ICMD_IF_LCMPLE:
2125
2126                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2127                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2128                         M_LCMP(s2, s1);
2129                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2130                         break;
2131
2132                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2133                 case ICMD_LRETURN:
2134
2135                         REPLACEMENT_POINT_RETURN(cd, iptr);
2136                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2137                         M_INTMOVE(s1, REG_RESULT);
2138                         goto nowperformreturn;
2139
2140                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2141
2142                         REPLACEMENT_POINT_RETURN(cd, iptr);
2143                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2144                         M_INTMOVE(s1, REG_RESULT);
2145
2146 #ifdef ENABLE_VERIFIER
2147                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2148                                 uc = iptr->sx.s23.s2.uc;
2149
2150                                 PROFILE_CYCLE_STOP;
2151
2152                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2153
2154                                 PROFILE_CYCLE_START;
2155                         }
2156 #endif /* ENABLE_VERIFIER */
2157                         goto nowperformreturn;
2158
2159                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2160                 case ICMD_DRETURN:
2161
2162                         REPLACEMENT_POINT_RETURN(cd, iptr);
2163                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2164                         M_FLTMOVE(s1, REG_FRESULT);
2165                         goto nowperformreturn;
2166
2167                 case ICMD_RETURN:      /* ...  ==> ...                                */
2168
2169                         REPLACEMENT_POINT_RETURN(cd, iptr);
2170
2171 nowperformreturn:
2172                         {
2173                         s4 i, p;
2174
2175                         p = cd->stackframesize;
2176
2177 #if !defined(NDEBUG)
2178                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2179                                 emit_verbosecall_exit(jd);
2180 #endif /* !defined(NDEBUG) */
2181
2182 #if defined(ENABLE_THREADS)
2183                         if (checksync && code_is_synchronized(code)) {
2184                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2185
2186                                 /* we need to save the proper return value */
2187                                 switch (iptr->opc) {
2188                                 case ICMD_IRETURN:
2189                                 case ICMD_ARETURN:
2190                                 case ICMD_LRETURN:
2191                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2192                                         break;
2193                                 case ICMD_FRETURN:
2194                                 case ICMD_DRETURN:
2195                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2196                                         break;
2197                                 }
2198
2199                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2200                                 M_CALL(REG_ITMP1);
2201
2202                                 /* and now restore the proper return value */
2203                                 switch (iptr->opc) {
2204                                 case ICMD_IRETURN:
2205                                 case ICMD_ARETURN:
2206                                 case ICMD_LRETURN:
2207                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2208                                         break;
2209                                 case ICMD_FRETURN:
2210                                 case ICMD_DRETURN:
2211                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2212                                         break;
2213                                 }
2214                         }
2215 #endif
2216
2217                         /* restore saved registers */
2218
2219                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2220                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2221                         }
2222                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2223                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2224                         }
2225
2226                         /* deallocate stack */
2227
2228                         if (cd->stackframesize)
2229                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2230
2231                         /* generate method profiling code */
2232
2233                         PROFILE_CYCLE_STOP;
2234
2235                         M_RET;
2236                         }
2237                         break;
2238
2239
2240                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2241                         {
2242                                 s4 i, l;
2243                                 branch_target_t *table;
2244
2245                                 table = iptr->dst.table;
2246
2247                                 l = iptr->sx.s23.s2.tablelow;
2248                                 i = iptr->sx.s23.s3.tablehigh;
2249
2250                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2251                                 M_INTMOVE(s1, REG_ITMP1);
2252
2253                                 if (l != 0)
2254                                         M_ISUB_IMM(l, REG_ITMP1);
2255
2256                                 /* number of targets */
2257                                 i = i - l + 1;
2258
2259                 /* range check */
2260
2261                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2262                                 emit_bugt(cd, table[0].block);
2263
2264                                 /* build jump table top down and use address of lowest entry */
2265
2266                                 table += i;
2267
2268                                 while (--i >= 0) {
2269                                         dseg_add_target(cd, table->block);
2270                                         --table;
2271                                 }
2272
2273                                 /* length of dataseg after last dseg_add_target is used
2274                                    by load */
2275
2276                                 M_MOV_IMM(0, REG_ITMP2);
2277                                 dseg_adddata(cd);
2278                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2279                                 M_JMP(REG_ITMP1);
2280                         }
2281                         break;
2282
2283
2284                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2285                         {
2286                                 s4 i;
2287                                 lookup_target_t *lookup;
2288
2289                                 lookup = iptr->dst.lookup;
2290
2291                                 i = iptr->sx.s23.s2.lookupcount;
2292
2293                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2294                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2295
2296                                 while (--i >= 0) {
2297                                         M_ICMP_IMM(lookup->value, s1);
2298                                         emit_beq(cd, lookup->target.block);
2299                                         lookup++;
2300                                 }
2301
2302                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2303                                 ALIGNCODENOP;
2304                         }
2305                         break;
2306
2307
2308                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2309
2310                         REPLACEMENT_POINT_FORGC_BUILTIN(cd, iptr);
2311
2312                         bte = iptr->sx.s23.s3.bte;
2313                         md  = bte->md;
2314                         goto gen_method;
2315
2316                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2317
2318                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2319                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2320                 case ICMD_INVOKEINTERFACE:
2321
2322                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2323
2324                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2325                                 lm = NULL;
2326                                 um = iptr->sx.s23.s3.um;
2327                                 md = um->methodref->parseddesc.md;
2328                         }
2329                         else {
2330                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2331                                 um = NULL;
2332                                 md = lm->parseddesc;
2333                         }
2334
2335 gen_method:
2336                         s3 = md->paramcount;
2337
2338                         MCODECHECK((20 * s3) + 128);
2339
2340                         /* copy arguments to registers or stack location */
2341
2342                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2343                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2344                                 d   = md->params[s3].regoff;
2345
2346                                 /* already preallocated (ARGVAR)? */
2347
2348                                 if (var->flags & PREALLOC)
2349                                         continue;
2350
2351                                 if (IS_INT_LNG_TYPE(var->type)) {
2352                                         if (!md->params[s3].inmemory) {
2353                                                 s1 = emit_load(jd, iptr, var, d);
2354                                                 M_INTMOVE(s1, d);
2355                                         }
2356                                         else {
2357                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2358                                                 M_LST(s1, REG_SP, d);
2359                                         }
2360                                 }
2361                                 else {
2362                                         if (!md->params[s3].inmemory) {
2363                                                 s1 = emit_load(jd, iptr, var, d);
2364                                                 M_FLTMOVE(s1, d);
2365                                         }
2366                                         else {
2367                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2368
2369                                                 if (IS_2_WORD_TYPE(var->type))
2370                                                         M_DST(s1, REG_SP, d);
2371                                                 else
2372                                                         M_FST(s1, REG_SP, d);
2373                                         }
2374                                 }
2375                         }
2376
2377                         /* generate method profiling code */
2378
2379                         PROFILE_CYCLE_STOP;
2380
2381                         switch (iptr->opc) {
2382                         case ICMD_BUILTIN:
2383                                 if (bte->stub == NULL) {
2384                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2385                                 }
2386                                 else {
2387                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2388                                 }
2389                                 M_CALL(REG_ITMP1);
2390                                 break;
2391
2392                         case ICMD_INVOKESPECIAL:
2393                                 emit_nullpointer_check(cd, iptr, REG_A0);
2394                                 /* fall through */
2395
2396                         case ICMD_INVOKESTATIC:
2397                                 if (lm == NULL) {
2398                                         disp = dseg_add_unique_address(cd, um);
2399
2400                                         patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
2401                                                                                   um, disp);
2402                                 }
2403                                 else {
2404                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2405                                 }
2406
2407                                 M_ALD(REG_ITMP2, RIP, disp);
2408                                 M_CALL(REG_ITMP2);
2409                                 break;
2410
2411                         case ICMD_INVOKEVIRTUAL:
2412                                 if (lm == NULL) {
2413                                         patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
2414
2415                                         s1 = 0;
2416                                 }
2417                                 else {
2418                                         s1 = OFFSET(vftbl_t, table[0]) +
2419                                                 sizeof(methodptr) * lm->vftblindex;
2420                                 }
2421
2422                                 /* implicit null-pointer check */
2423                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2424                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2425                                 M_CALL(REG_ITMP3);
2426                                 break;
2427
2428                         case ICMD_INVOKEINTERFACE:
2429                                 if (lm == NULL) {
2430                                         patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
2431
2432                                         s1 = 0;
2433                                         s2 = 0;
2434                                 }
2435                                 else {
2436                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2437                                                 sizeof(methodptr) * lm->clazz->index;
2438
2439                                         s2 = sizeof(methodptr) * (lm - lm->clazz->methods);
2440                                 }
2441
2442                                 /* implicit null-pointer check */
2443                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2444                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2445                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2446                                 M_CALL(REG_ITMP3);
2447                                 break;
2448                         }
2449
2450                         /* generate method profiling code */
2451
2452                         PROFILE_CYCLE_START;
2453
2454                         /* store size of call code in replacement point */
2455
2456                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2457                         REPLACEMENT_POINT_FORGC_BUILTIN_RETURN(cd, iptr);
2458
2459                         /* store return value */
2460
2461                         switch (md->returntype.type) {
2462                         case TYPE_INT:
2463                         case TYPE_LNG:
2464                         case TYPE_ADR:
2465                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2466                                 M_INTMOVE(REG_RESULT, s1);
2467                                 emit_store_dst(jd, iptr, s1);
2468                                 break;
2469                         case TYPE_FLT:
2470                         case TYPE_DBL:
2471                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2472                                 M_FLTMOVE(REG_FRESULT, s1);
2473                                 emit_store_dst(jd, iptr, s1);
2474                                 break;
2475                         default:
2476                                 /* TYPE_VOID */
2477                                 break;
2478                         }
2479                         break;
2480
2481
2482                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2483
2484                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2485                                 /* object type cast-check */
2486
2487                                 classinfo *super;
2488                                 s4         superindex;
2489
2490                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2491                                         super      = NULL;
2492                                         superindex = 0;
2493                                 }
2494                                 else {
2495                                         super      = iptr->sx.s23.s3.c.cls;
2496                                         superindex = super->index;
2497                                 }
2498
2499                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2500
2501                                 /* if class is not resolved, check which code to call */
2502
2503                                 if (super == NULL) {
2504                                         M_TEST(s1);
2505                                         emit_label_beq(cd, BRANCH_LABEL_1);
2506
2507                                         patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2508                                                                                   iptr->sx.s23.s3.c.ref, 0);
2509
2510                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2511                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2512                                         emit_label_beq(cd, BRANCH_LABEL_2);
2513                                 }
2514
2515                                 /* interface checkcast code */
2516
2517                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2518                                         if (super != NULL) {
2519                                                 M_TEST(s1);
2520                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2521                                         }
2522
2523                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2524
2525                                         if (super == NULL) {
2526                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
2527                                                                                           iptr->sx.s23.s3.c.ref,
2528                                                                                           0);
2529                                         }
2530
2531                                         M_ILD32(REG_ITMP3,
2532                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2533                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2534                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2535
2536                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2537                                                         OFFSET(vftbl_t, interfacetable[0]) -
2538                                                         superindex * sizeof(methodptr*));
2539                                         M_TEST(REG_ITMP3);
2540                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2541
2542                                         if (super == NULL)
2543                                                 emit_label_br(cd, BRANCH_LABEL_4);
2544                                         else
2545                                                 emit_label(cd, BRANCH_LABEL_3);
2546                                 }
2547
2548                                 /* class checkcast code */
2549
2550                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2551                                         if (super == NULL) {
2552                                                 emit_label(cd, BRANCH_LABEL_2);
2553
2554                                                 cr   = iptr->sx.s23.s3.c.ref;
2555                                                 disp = dseg_add_unique_address(cd, cr);
2556
2557                                                 patcher_add_patch_ref(jd,
2558                                                                                           PATCHER_resolve_classref_to_vftbl,
2559                                                                                           cr, disp);
2560                                         }
2561                                         else {
2562                                                 M_TEST(s1);
2563                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2564
2565                                                 disp = dseg_add_address(cd, super->vftbl);
2566                                         }
2567
2568                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2569                                         M_ALD(REG_ITMP3, RIP, disp);
2570
2571                                         if (super == NULL || super->vftbl->subtype_depth >= DISPLAY_SIZE) {
2572                                                 M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_offset));
2573
2574                                                 *(cd->mcodeptr++) = 0x4d;
2575                                                 *(cd->mcodeptr++) = 0x3b;
2576                                                 *(cd->mcodeptr++) = 0x1c;
2577                                                 *(cd->mcodeptr++) = 0x02;
2578                                                 /* cmp (ITMP2, ITMP1, 1), ITMP3 */
2579
2580                                                 emit_label_beq(cd, BRANCH_LABEL_6);  /* good */
2581
2582                                                 if (super == NULL) {
2583                                                         M_LCMP_IMM(OFFSET(vftbl_t, subtype_display[DISPLAY_SIZE]), REG_ITMP1);
2584                                                         emit_label_bne(cd, BRANCH_LABEL_10);  /* throw */
2585                                                 }
2586
2587                                                 M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_depth));
2588                                                 *(cd->mcodeptr++) = 0x41;
2589                                                 *(cd->mcodeptr++) = 0x39;
2590                                                 *(cd->mcodeptr++) = 0x42;
2591                                                 *(cd->mcodeptr++) = OFFSET(vftbl_t, subtype_depth);
2592                                                 /* cmpl ITMP1, subtype_depth(ITMP2) */
2593                                                 emit_label_blt(cd, BRANCH_LABEL_9);  /* throw */
2594
2595                                                 M_ALD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, subtype_overflow));
2596
2597                                                 *(cd->mcodeptr++) = 0x4d;
2598                                                 *(cd->mcodeptr++) = 0x3b;
2599                                                 *(cd->mcodeptr++) = 0x5c;
2600                                                 *(cd->mcodeptr++) = 0xc2;
2601                                                 *(cd->mcodeptr++) = -8*DISPLAY_SIZE;
2602                                                 /* cmp -8*DISPL(ITMP2, ITMP1, 8), ITMP3 */
2603
2604                                                 emit_label_beq(cd, BRANCH_LABEL_7);  /* good */
2605
2606                                                 emit_label(cd, BRANCH_LABEL_9);
2607                                                 if (super == NULL)
2608                                                         emit_label(cd, BRANCH_LABEL_10);
2609
2610                                                 /* reload s1, might have been destroyed */
2611                                                 emit_load_s1(jd, iptr, REG_ITMP1);
2612                                                 M_ALD_MEM(s1, TRAP_ClassCastException);
2613
2614                                                 emit_label(cd, BRANCH_LABEL_7);
2615                                                 emit_label(cd, BRANCH_LABEL_6);
2616                                                 /* reload s1, might have been destroyed */
2617                                                 emit_load_s1(jd, iptr, REG_ITMP1);
2618                                         }
2619                                         else {
2620                                                 assert(super->vftbl->subtype_offset < 0x80);
2621                                                 *(cd->mcodeptr++) = 0x4d;
2622                                                 *(cd->mcodeptr++) = 0x3b;
2623                                                 *(cd->mcodeptr++) = 0x5a;
2624                                                 *(cd->mcodeptr++) = super->vftbl->subtype_offset;
2625                                                 /* cmp off(ITMP2), ITMP3 */
2626
2627                                                 emit_classcast_check(cd, iptr, BRANCH_NE, REG_ITMP3, s1);
2628                                         }
2629
2630                                         if (super != NULL)
2631                                                 emit_label(cd, BRANCH_LABEL_5);
2632                                 }
2633
2634                                 if (super == NULL) {
2635                                         emit_label(cd, BRANCH_LABEL_1);
2636                                         emit_label(cd, BRANCH_LABEL_4);
2637                                 }
2638
2639                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2640                         }
2641                         else {
2642                                 /* array type cast-check */
2643
2644                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2645                                 M_INTMOVE(s1, REG_A0);
2646
2647                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2648                                         cr   = iptr->sx.s23.s3.c.ref;
2649                                         disp = dseg_add_unique_address(cd, cr);
2650
2651                                         patcher_add_patch_ref(jd,
2652                                                                                   PATCHER_resolve_classref_to_classinfo,
2653                                                                                   cr, disp);
2654                                 }
2655                                 else {
2656                                         disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2657                                 }
2658
2659                                 M_ALD(REG_A1, RIP, disp);
2660                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2661                                 M_CALL(REG_ITMP1);
2662
2663                                 /* s1 may have been destroyed over the function call */
2664                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2665                                 M_TEST(REG_RESULT);
2666                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2667
2668                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2669                         }
2670
2671                         M_INTMOVE(s1, d);
2672                         emit_store_dst(jd, iptr, d);
2673                         break;
2674
2675                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2676
2677                         {
2678                         classinfo *super;
2679                         s4         superindex;
2680
2681                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2682                                 super      = NULL;
2683                                 superindex = 0;
2684                         }
2685                         else {
2686                                 super      = iptr->sx.s23.s3.c.cls;
2687                                 superindex = super->index;
2688                         }
2689
2690                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2691                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2692
2693                         if (s1 == d) {
2694                                 M_INTMOVE(s1, REG_ITMP1);
2695                                 s1 = REG_ITMP1;
2696                         }
2697
2698                         M_CLR(d);
2699
2700                         /* if class is not resolved, check which code to call */
2701
2702                         if (super == NULL) {
2703                                 M_TEST(s1);
2704                                 emit_label_beq(cd, BRANCH_LABEL_1);
2705
2706                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2707                                                                           iptr->sx.s23.s3.c.ref, 0);
2708
2709                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2710                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2711                                 emit_label_beq(cd, BRANCH_LABEL_2);
2712                         }
2713
2714                         /* interface instanceof code */
2715
2716                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2717                                 if (super != NULL) {
2718                                         M_TEST(s1);
2719                                         emit_label_beq(cd, BRANCH_LABEL_3);
2720                                 }
2721
2722                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2723
2724                                 if (super == NULL) {
2725                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
2726                                                                                   iptr->sx.s23.s3.c.ref, 0);
2727                                 }
2728
2729                                 M_ILD32(REG_ITMP3,
2730                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2731                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2732
2733                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2734
2735                                 M_BLE(a);
2736                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2737                                                 OFFSET(vftbl_t, interfacetable[0]) -
2738                                                 superindex * sizeof(methodptr*));
2739                                 M_TEST(REG_ITMP1);
2740                                 M_SETNE(d);
2741
2742                                 if (super == NULL)
2743                                         emit_label_br(cd, BRANCH_LABEL_4);
2744                                 else
2745                                         emit_label(cd, BRANCH_LABEL_3);
2746                         }
2747
2748                         /* class instanceof code */
2749
2750                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2751                                 if (super == NULL) {
2752                                         emit_label(cd, BRANCH_LABEL_2);
2753
2754                                         cr   = iptr->sx.s23.s3.c.ref;
2755                                         disp = dseg_add_unique_address(cd, cr);
2756
2757                                         patcher_add_patch_ref(jd,
2758                                                                                   PATCHER_resolve_classref_to_vftbl,
2759                                                                                   cr, disp);
2760                                 }
2761                                 else {
2762                                         M_TEST(s1);
2763                                         emit_label_beq(cd, BRANCH_LABEL_5);
2764
2765                                         disp = dseg_add_address(cd, super->vftbl);
2766                                 }
2767
2768                                 M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2769                                 M_ALD(REG_ITMP3, RIP, disp);
2770
2771                                 if (super == NULL || super->vftbl->subtype_depth >= DISPLAY_SIZE) {
2772                                         M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_offset));
2773
2774                                         *(cd->mcodeptr++) = 0x4d;
2775                                         *(cd->mcodeptr++) = 0x3b;
2776                                         *(cd->mcodeptr++) = 0x1c;
2777                                         *(cd->mcodeptr++) = 0x02;
2778                                         /* cmp (ITMP2, ITMP1, 1), ITMP3 */
2779
2780                                         emit_label_bne(cd, BRANCH_LABEL_8); /* jump over INC/SETE */
2781                                         if (d == REG_ITMP2) {
2782                                                 M_SETE(d);
2783                                                 M_BSEXT(d, d);
2784                                         } else
2785                                                 M_LINC(d);
2786                                         emit_label_br(cd, BRANCH_LABEL_6);  /* true */
2787                                         emit_label(cd, BRANCH_LABEL_8);
2788
2789                                         if (super == NULL) {
2790                                                 M_LCMP_IMM(OFFSET(vftbl_t, subtype_display[DISPLAY_SIZE]), REG_ITMP1);
2791                                                 emit_label_bne(cd, BRANCH_LABEL_10);  /* false */
2792                                         }
2793
2794                                         M_ILD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_depth));
2795                                         *(cd->mcodeptr++) = 0x41;
2796                                         *(cd->mcodeptr++) = 0x39;
2797                                         *(cd->mcodeptr++) = 0x42;
2798                                         *(cd->mcodeptr++) = OFFSET(vftbl_t, subtype_depth);
2799                                         /* cmpl ITMP1, subtype_depth(ITMP2) */
2800                                         emit_label_blt(cd, BRANCH_LABEL_9);  /* false */
2801
2802                                         M_ALD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, subtype_overflow));
2803
2804                                         *(cd->mcodeptr++) = 0x4d;
2805                                         *(cd->mcodeptr++) = 0x3b;
2806                                         *(cd->mcodeptr++) = 0x5c;
2807                                         *(cd->mcodeptr++) = 0xc2;
2808                                         *(cd->mcodeptr++) = -8*DISPLAY_SIZE;
2809                                         /* cmp -8*DISPL(ITMP2, ITMP1, 8), ITMP3 */
2810
2811                                         M_SETE(d);
2812                                         if (d == REG_ITMP2) {
2813                                                 M_BSEXT(d, d);
2814
2815                                                 emit_label_br(cd, BRANCH_LABEL_7); /* jump over M_CLR */
2816                                         }
2817
2818                                         emit_label(cd, BRANCH_LABEL_9);
2819                                         if (super == NULL)
2820                                                 emit_label(cd, BRANCH_LABEL_10);
2821                                         if (d == REG_ITMP2) {
2822                                                 M_CLR(d);
2823
2824                                                 emit_label(cd, BRANCH_LABEL_7);
2825                                         }
2826                                         emit_label(cd, BRANCH_LABEL_6);
2827                                 }
2828                                 else {
2829                                         assert(super->vftbl->subtype_offset < 0x80);
2830                                         *(cd->mcodeptr++) = 0x4d;
2831                                         *(cd->mcodeptr++) = 0x3b;
2832                                         *(cd->mcodeptr++) = 0x5a;
2833                                         *(cd->mcodeptr++) = super->vftbl->subtype_offset;
2834                                         /* cmp off(ITMP2), ITMP3 */
2835
2836                                         M_SETE(d);
2837                                         if (d == REG_ITMP2)
2838                                                 M_BSEXT(d, d);
2839                                 }
2840
2841                                 if (super != NULL)
2842                                         emit_label(cd, BRANCH_LABEL_5);
2843                         }
2844
2845                         if (super == NULL) {
2846                                 emit_label(cd, BRANCH_LABEL_1);
2847                                 emit_label(cd, BRANCH_LABEL_4);
2848                         }
2849
2850                         emit_store_dst(jd, iptr, d);
2851                         }
2852                         break;
2853
2854                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2855
2856                         /* check for negative sizes and copy sizes to stack if necessary  */
2857
2858                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2859
2860                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2861
2862                                 /* copy SAVEDVAR sizes to stack */
2863                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2864
2865                                 /* Already Preallocated? */
2866                                 if (!(var->flags & PREALLOC)) {
2867                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2868                                         M_LST(s2, REG_SP, s1 * 8);
2869                                 }
2870                         }
2871
2872                         /* a0 = dimension count */
2873
2874                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2875
2876                         /* is a patcher function set? */
2877
2878                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2879                                 cr   = iptr->sx.s23.s3.c.ref;
2880                                 disp = dseg_add_unique_address(cd, cr);
2881
2882                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
2883                                                                           cr, disp);
2884                         }
2885                         else {
2886                                 disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2887                         }
2888
2889                         /* a1 = classinfo */
2890
2891                         M_ALD(REG_A1, RIP, disp);
2892
2893                         /* a2 = pointer to dimensions = stack pointer */
2894
2895                         M_MOV(REG_SP, REG_A2);
2896
2897                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2898                         M_CALL(REG_ITMP1);
2899
2900                         /* check for exception before result assignment */
2901
2902                         emit_exception_check(cd, iptr);
2903
2904                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2905                         M_INTMOVE(REG_RESULT, s1);
2906                         emit_store_dst(jd, iptr, s1);
2907                         break;
2908
2909                 default:
2910                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2911                                                                                    iptr->opc);
2912                         return false;
2913         } /* switch */
2914
2915         } /* for instruction */
2916
2917         MCODECHECK(512); /* XXX require a lower number? */
2918
2919         /* At the end of a basic block we may have to append some nops,
2920            because the patcher stub calling code might be longer than the
2921            actual instruction. So codepatching does not change the
2922            following block unintentionally. */
2923
2924         if (cd->mcodeptr < cd->lastmcodeptr) {
2925                 while (cd->mcodeptr < cd->lastmcodeptr) {
2926                         M_NOP;
2927                 }
2928         }
2929
2930         } /* if (bptr -> flags >= BBREACHED) */
2931         } /* for basic block */
2932
2933         /* Generate patcher traps. */
2934
2935         emit_patcher_traps(jd);
2936
2937         /* everything's ok */
2938
2939         return true;
2940 }
2941
2942
2943 /* codegen_emit_stub_native ****************************************************
2944
2945    Emits a stub routine which calls a native method.
2946
2947 *******************************************************************************/
2948
2949 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f, int skipparams)
2950 {
2951         methodinfo  *m;
2952         codeinfo    *code;
2953         codegendata *cd;
2954         methoddesc  *md;
2955         int          i, j;
2956         int          s1, s2;
2957         int          disp;
2958
2959         /* Sanity check. */
2960
2961         assert(f != NULL);
2962
2963         /* Get required compiler data. */
2964
2965         m    = jd->m;
2966         code = jd->code;
2967         cd   = jd->cd;
2968
2969         /* initialize variables */
2970
2971         md = m->parseddesc;
2972
2973         /* calculate stack frame size */
2974
2975         cd->stackframesize =
2976                 sizeof(stackframeinfo_t) / SIZEOF_VOID_P +
2977                 sizeof(localref_table) / SIZEOF_VOID_P +
2978                 md->paramcount +
2979                 (md->returntype.type == TYPE_VOID ? 0 : 1) +
2980                 nmd->memuse;
2981
2982         ALIGN_ODD(cd->stackframesize);              /* keep stack 16-byte aligned */
2983
2984         /* create method header */
2985
2986         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2987         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2988         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2989         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2990         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2991
2992 #if defined(ENABLE_PROFILING)
2993         /* generate native method profiling code */
2994
2995         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
2996                 /* count frequency */
2997
2998                 M_MOV_IMM(code, REG_ITMP3);
2999                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3000         }
3001 #endif
3002
3003         /* generate stub code */
3004
3005         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3006
3007 #if defined(ENABLE_GC_CACAO)
3008         /* Save callee saved integer registers in stackframeinfo (GC may
3009            need to recover them during a collection). */
3010
3011         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
3012                 OFFSET(stackframeinfo_t, intregs);
3013
3014         for (i = 0; i < INT_SAV_CNT; i++)
3015                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3016 #endif
3017
3018         /* save integer and float argument registers */
3019
3020         for (i = 0; i < md->paramcount; i++) {
3021                 if (!md->params[i].inmemory) {
3022                         s1 = md->params[i].regoff;
3023
3024                         switch (md->paramtypes[i].type) {
3025                         case TYPE_INT:
3026                         case TYPE_LNG:
3027                         case TYPE_ADR:
3028                                 M_LST(s1, REG_SP, i * 8);
3029                                 break;
3030                         case TYPE_FLT:
3031                         case TYPE_DBL:
3032                                 M_DST(s1, REG_SP, i * 8);
3033                                 break;
3034                         }
3035                 }
3036         }
3037
3038         /* create dynamic stack info */
3039
3040         M_MOV(REG_SP, REG_A0);
3041         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3042         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3043         M_CALL(REG_ITMP1);
3044
3045         /* remember class argument */
3046
3047         if (m->flags & ACC_STATIC)
3048                 M_MOV(REG_RESULT, REG_ITMP2);
3049
3050         /* restore integer and float argument registers */
3051
3052         for (i = 0; i < md->paramcount; i++) {
3053                 if (!md->params[i].inmemory) {
3054                         s1 = md->params[i].regoff;
3055
3056                         switch (md->paramtypes[i].type) {
3057                         case TYPE_INT:
3058                         case TYPE_LNG:
3059                         case TYPE_ADR:
3060                                 M_LLD(s1, REG_SP, i * 8);
3061                                 break;
3062                         case TYPE_FLT:
3063                         case TYPE_DBL:
3064                                 M_DLD(s1, REG_SP, i * 8);
3065                                 break;
3066                         }
3067                 }
3068         }
3069
3070         /* Copy or spill arguments to new locations. */
3071
3072         for (i = md->paramcount - 1, j = i + skipparams; i >= 0; i--, j--) {
3073                 s2 = nmd->params[j].regoff;
3074
3075                 switch (md->paramtypes[i].type) {
3076                 case TYPE_INT:
3077                 case TYPE_LNG:
3078                 case TYPE_ADR:
3079                         if (!md->params[i].inmemory) {
3080                                 s1 = md->params[i].regoff;
3081
3082                                 if (!nmd->params[j].inmemory)
3083                                         M_INTMOVE(s1, s2);
3084                                 else
3085                                         M_LST(s1, REG_SP, s2);
3086                         }
3087                         else {
3088                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3089                                 M_LLD(REG_ITMP1, REG_SP, s1);
3090                                 M_LST(REG_ITMP1, REG_SP, s2);
3091                         }
3092                         break;
3093                 case TYPE_FLT:
3094                         /* We only copy spilled float arguments, as the float
3095                            argument registers keep unchanged. */
3096
3097                         if (md->params[i].inmemory) {
3098                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3099
3100                                 M_FLD(REG_FTMP1, REG_SP, s1);
3101                                 M_FST(REG_FTMP1, REG_SP, s2);
3102                         }
3103                         break;
3104                 case TYPE_DBL:
3105                         if (md->params[i].inmemory) {
3106                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3107                                 M_DLD(REG_FTMP1, REG_SP, s1);
3108                                 M_DST(REG_FTMP1, REG_SP, s2);
3109                         }
3110                         break;
3111                 }
3112         }
3113
3114         /* Handle native Java methods. */
3115
3116         if (m->flags & ACC_NATIVE) {
3117                 /* put class into second argument register */
3118
3119                 if (m->flags & ACC_STATIC)
3120                         M_MOV(REG_ITMP2, REG_A1);
3121
3122                 /* put env into first argument register */
3123
3124                 M_MOV_IMM(_Jv_env, REG_A0);
3125         }
3126
3127         /* Call the native function. */
3128
3129         disp = dseg_add_functionptr(cd, f);
3130         M_ALD(REG_ITMP1, RIP, disp);
3131         M_CALL(REG_ITMP1);
3132
3133         /* save return value */
3134
3135         switch (md->returntype.type) {
3136         case TYPE_INT:
3137         case TYPE_LNG:
3138         case TYPE_ADR:
3139                 switch (md->returntype.decltype) {
3140                 case PRIMITIVETYPE_BOOLEAN:
3141                         M_BZEXT(REG_RESULT, REG_RESULT);
3142                         break;
3143                 case PRIMITIVETYPE_BYTE:
3144                         M_BSEXT(REG_RESULT, REG_RESULT);
3145                         break;
3146                 case PRIMITIVETYPE_CHAR:
3147                         M_CZEXT(REG_RESULT, REG_RESULT);
3148                         break;
3149                 case PRIMITIVETYPE_SHORT:
3150                         M_SSEXT(REG_RESULT, REG_RESULT);
3151                         break;
3152                 }
3153                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3154                 break;
3155         case TYPE_FLT:
3156         case TYPE_DBL:
3157                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3158                 break;
3159         case TYPE_VOID:
3160                 break;
3161         }
3162
3163         /* remove native stackframe info */
3164
3165         M_MOV(REG_SP, REG_A0);
3166         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3167         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3168         M_CALL(REG_ITMP1);
3169         M_MOV(REG_RESULT, REG_ITMP3);
3170
3171         /* restore return value */
3172
3173         switch (md->returntype.type) {
3174         case TYPE_INT:
3175         case TYPE_LNG:
3176         case TYPE_ADR:
3177                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3178                 break;
3179         case TYPE_FLT:
3180         case TYPE_DBL:
3181                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3182                 break;
3183         case TYPE_VOID:
3184                 break;
3185         }
3186
3187 #if defined(ENABLE_GC_CACAO)
3188         /* Restore callee saved integer registers from stackframeinfo (GC
3189            might have modified them during a collection). */
3190
3191         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
3192                 OFFSET(stackframeinfo_t, intregs);
3193
3194         for (i = 0; i < INT_SAV_CNT; i++)
3195                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3196 #endif
3197
3198         /* remove stackframe */
3199
3200         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3201
3202         /* test for exception */
3203
3204         M_TEST(REG_ITMP3);
3205         M_BNE(1);
3206         M_RET;
3207
3208         /* handle exception */
3209
3210         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3211         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3212         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3213
3214         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3215         M_JMP(REG_ITMP3);
3216 }
3217
3218
3219 /*
3220  * These are local overrides for various environment variables in Emacs.
3221  * Please do not remove this and leave it at the end of the file, where
3222  * Emacs will automagically detect them.
3223  * ---------------------------------------------------------------------
3224  * Local variables:
3225  * mode: c
3226  * indent-tabs-mode: t
3227  * c-basic-offset: 4
3228  * tab-width: 4
3229  * End:
3230  * vim:noexpandtab:sw=4:ts=4:
3231  */