src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25 */
  26
  27
  28 #include "config.h"
  29
  30 #include <assert.h>
  31 #include <stdio.h>
  32 #include <stdint.h>
  33
  34 #include "vm/types.h"
  35
  36 #include "md-abi.h"
  37
  38 #include "vm/jit/x86_64/arch.h"
  39 #include "vm/jit/x86_64/codegen.h"
  40 #include "vm/jit/x86_64/emit.h"
  41
  42 #include "mm/memory.h"
  43
  44 #include "native/jni.h"
  45 #include "native/localref.h"
  46 #include "native/native.h"
  47
  48 #include "threads/lock-common.h"
  49
  50 #include "vm/builtin.h"
  51 #include "vm/exceptions.h"
  52 #include "vm/global.h"
  53 #include "vm/primitive.h"
  54 #include "vm/stringlocal.h"
  55 #include "vm/vm.h"
  56
  57 #include "vm/jit/abi.h"
  58 #include "vm/jit/asmpart.h"
  59 #include "vm/jit/code.h"
  60 #include "vm/jit/codegen-common.h"
  61 #include "vm/jit/dseg.h"
  62 #include "vm/jit/emit-common.h"
  63 #include "vm/jit/jit.h"
  64 #include "vm/jit/linenumbertable.h"
  65 #include "vm/jit/methodheader.h"
  66 #include "vm/jit/parse.h"
  67 #include "vm/jit/patcher-common.h"
  68 #include "vm/jit/reg.h"
  69 #include "vm/jit/replace.h"
  70 #include "vm/jit/stacktrace.h"
  71 #include "vm/jit/trap.h"
  72
  73 #if defined(ENABLE_LSRA)
  74 # include "vm/jit/allocator/lsra.h"
  75 #endif
  76
  77 #include "vmcore/loader.h"
  78 #include "vmcore/options.h"
  79 #include "vmcore/statistics.h"
  80
  81
  82 /* codegen_emit ****************************************************************
  83
  84    Generates machine code.
  85
  86 *******************************************************************************/
  87
  88 bool codegen_emit(jitdata *jd)
  89 {
  90         methodinfo         *m;
  91         codeinfo           *code;
  92         codegendata        *cd;
  93         registerdata       *rd;
  94         s4                  len, s1, s2, s3, d, disp;
  95         u2                  currentline;
  96         ptrint              a;
  97         varinfo            *var, *dst;
  98         basicblock         *bptr;
  99         instruction        *iptr;
 100         constant_classref  *cr;
 101         unresolved_class   *uc;
 102         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 103         unresolved_method  *um;
 104         builtintable_entry *bte;
 105         methoddesc         *md;
 106         fieldinfo          *fi;
 107         unresolved_field   *uf;
 108         s4                  fieldtype;
 109         s4                 varindex;
 110
 111         /* get required compiler data */
 112
 113         m    = jd->m;
 114         code = jd->code;
 115         cd   = jd->cd;
 116         rd   = jd->rd;
 117
 118         /* prevent compiler warnings */
 119
 120         d   = 0;
 121         lm  = NULL;
 122         um  = NULL;
 123         bte = NULL;
 124
 125         {
 126         s4 i, p, t, l;
 127         s4 savedregs_num;
 128
 129         savedregs_num = 0;
 130
 131         /* space to save used callee saved registers */
 132
 133         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 134         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 135
 136         cd->stackframesize = rd->memuse + savedregs_num;
 137
 138 #if defined(ENABLE_THREADS)
 139         /* space to save argument of monitor_enter */
 140
 141         if (checksync && code_is_synchronized(code))
 142                 cd->stackframesize++;
 143 #endif
 144
 145         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 146            native code e.g. libc or jni (alignment problems with
 147            movaps). */
 148
 149         if (!code_is_leafmethod(code) || opt_verbosecall)
 150                 cd->stackframesize |= 0x1;
 151
 152         /* create method header */
 153
 154         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 155         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 156
 157         code->synchronizedoffset = rd->memuse * 8;
 158
 159         if (code_is_leafmethod(code))
 160                 (void) dseg_add_unique_s4(cd, 1);                  /* IsLeaf          */
 161         else
 162                 (void) dseg_add_unique_s4(cd, 0);                  /* IsLeaf          */
 163
 164         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 165         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 166
 167 #if defined(ENABLE_PROFILING)
 168         /* generate method profiling code */
 169
 170         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 171                 /* count frequency */
 172
 173                 M_MOV_IMM(code, REG_ITMP3);
 174                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 175
 176                 PROFILE_CYCLE_START;
 177         }
 178 #endif
 179
 180         /* create stack frame (if necessary) */
 181
 182         if (cd->stackframesize)
 183                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 184
 185         /* save used callee saved registers */
 186
 187         p = cd->stackframesize;
 188         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 189                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 190         }
 191         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 192                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 193         }
 194
 195         /* take arguments out of register or stack frame */
 196
 197         md = m->parseddesc;
 198
 199         for (p = 0, l = 0; p < md->paramcount; p++) {
 200                 t = md->paramtypes[p].type;
 201
 202                 varindex = jd->local_map[l * 5 + t];
 203
 204                 l++;
 205                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 206                         l++;
 207
 208                 if (varindex == UNUSED)
 209                         continue;
 210
 211                 var = VAR(varindex);
 212
 213                 s1 = md->params[p].regoff;
 214
 215                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 216                         if (!md->params[p].inmemory) {           /* register arguments    */
 217                                 if (!IS_INMEMORY(var->flags))
 218                                         M_INTMOVE(s1, var->vv.regoff);
 219                                 else
 220                                     M_LST(s1, REG_SP, var->vv.regoff);
 221                         }
 222                         else {                                 /* stack arguments       */
 223                                 if (!IS_INMEMORY(var->flags))
 224                                         /* + 8 for return address */
 225                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 226                                 else
 227                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 228                         }
 229                 }
 230                 else {                                     /* floating args         */
 231                         if (!md->params[p].inmemory) {           /* register arguments    */
 232                                 if (!IS_INMEMORY(var->flags))
 233                                         M_FLTMOVE(s1, var->vv.regoff);
 234                                 else
 235                                         M_DST(s1, REG_SP, var->vv.regoff);
 236                         }
 237                         else {                                 /* stack arguments       */
 238                                 if (!IS_INMEMORY(var->flags))
 239                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 240                                 else
 241                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 242                         }
 243                 }
 244         }
 245
 246         /* save monitorenter argument */
 247
 248 #if defined(ENABLE_THREADS)
 249         if (checksync && code_is_synchronized(code)) {
 250                 /* stack offset for monitor argument */
 251
 252                 s1 = rd->memuse;
 253
 254                 if (opt_verbosecall) {
 255                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 256
 257                         for (p = 0; p < INT_ARG_CNT; p++)
 258                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 259
 260                         for (p = 0; p < FLT_ARG_CNT; p++)
 261                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 262
 263                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 264                 }
 265
 266                 /* decide which monitor enter function to call */
 267
 268                 if (m->flags & ACC_STATIC) {
 269                         M_MOV_IMM(&m->clazz->object.header, REG_A0);
 270                 }
 271                 else {
 272                         M_TEST(REG_A0);
 273                         M_BNE(8);
 274                         M_ALD_MEM(REG_A0, TRAP_NullPointerException);
 275                 }
 276
 277                 M_AST(REG_A0, REG_SP, s1 * 8);
 278                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 279                 M_CALL(REG_ITMP1);
 280
 281                 if (opt_verbosecall) {
 282                         for (p = 0; p < INT_ARG_CNT; p++)
 283                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 284
 285                         for (p = 0; p < FLT_ARG_CNT; p++)
 286                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 287
 288                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 289                 }
 290         }
 291 #endif
 292
 293 #if !defined(NDEBUG)
 294         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 295                 emit_verbosecall_enter(jd);
 296 #endif /* !defined(NDEBUG) */
 297
 298         }
 299
 300         /* end of header generation */
 301
 302         /* create replacement points */
 303
 304         REPLACEMENT_POINTS_INIT(cd, jd);
 305
 306         /* walk through all basic blocks */
 307
 308         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 309
 310                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 311
 312                 if (bptr->flags >= BBREACHED) {
 313
 314                 /* branch resolving */
 315
 316                 codegen_resolve_branchrefs(cd, bptr);
 317
 318                 /* handle replacement points */
 319
 320                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 321
 322                 /* copy interface registers to their destination */
 323
 324                 len = bptr->indepth;
 325                 MCODECHECK(512);
 326
 327 #if defined(ENABLE_PROFILING)
 328                 /* generate basicblock profiling code */
 329
 330                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 331                         /* count frequency */
 332
 333                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 334                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 335
 336                         /* if this is an exception handler, start profiling again */
 337
 338                         if (bptr->type == BBTYPE_EXH)
 339                                 PROFILE_CYCLE_START;
 340                 }
 341 #endif
 342
 343 #if defined(ENABLE_LSRA)
 344                 if (opt_lsra) {
 345                         while (len) {
 346                                 len--;
 347                                 src = bptr->invars[len];
 348                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 349                                         if (bptr->type == BBTYPE_EXH) {
 350 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 351                                                 if (!IS_INMEMORY(src->flags))
 352                                                         d= src->vv.regoff;
 353                                                 else
 354                                                         d=REG_ITMP1;
 355                                                 M_INTMOVE(REG_ITMP1, d);
 356                                                 emit_store(jd, NULL, src, d);
 357                                         }
 358                                 }
 359                         }
 360
 361                 } else {
 362 #endif
 363
 364                 while (len) {
 365                         len--;
 366                         var = VAR(bptr->invars[len]);
 367                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 368                                 if (bptr->type == BBTYPE_EXH) {
 369                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 370                                         M_INTMOVE(REG_ITMP1, d);
 371                                         emit_store(jd, NULL, var, d);
 372                                 }
 373                         }
 374                         else {
 375                                 assert((var->flags & INOUT));
 376                         }
 377                 }
 378 #if defined(ENABLE_LSRA)
 379                 }
 380 #endif
 381                 /* walk through all instructions */
 382
 383                 len = bptr->icount;
 384                 currentline = 0;
 385
 386                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 387                         if (iptr->line != currentline) {
 388                                 linenumbertable_list_entry_add(cd, iptr->line);
 389                                 currentline = iptr->line;
 390                         }
 391
 392                         MCODECHECK(1024);                         /* 1KB should be enough */
 393
 394                 switch (iptr->opc) {
 395                 case ICMD_NOP:        /* ...  ==> ...                                 */
 396                 case ICMD_POP:        /* ..., value  ==> ...                          */
 397                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 398                         break;
 399
 400                 case ICMD_INLINE_START:
 401
 402                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 403                         break;
 404
 405                 case ICMD_INLINE_BODY:
 406
 407                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 408                         linenumbertable_list_entry_add_inline_start(cd, iptr);
 409                         linenumbertable_list_entry_add(cd, iptr->line);
 410                         break;
 411
 412                 case ICMD_INLINE_END:
 413
 414                         linenumbertable_list_entry_add_inline_end(cd, iptr);
 415                         linenumbertable_list_entry_add(cd, iptr->line);
 416                         break;
 417
 418                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 419
 420                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 421                         emit_nullpointer_check(cd, iptr, s1);
 422                         break;
 423
 424                 /* constant operations ************************************************/
 425
 426                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 427
 428                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 429                         ICONST(d, iptr->sx.val.i);
 430                         emit_store_dst(jd, iptr, d);
 431                         break;
 432
 433                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 434
 435                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 436                         LCONST(d, iptr->sx.val.l);
 437                         emit_store_dst(jd, iptr, d);
 438                         break;
 439
 440                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 441
 442                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 443                         disp = dseg_add_float(cd, iptr->sx.val.f);
 444                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 445                         emit_store_dst(jd, iptr, d);
 446                         break;
 447
 448                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 449
 450                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 451                         disp = dseg_add_double(cd, iptr->sx.val.d);
 452                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 453                         emit_store_dst(jd, iptr, d);
 454                         break;
 455
 456                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 457
 458                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 459
 460                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 461                                 cr   = iptr->sx.val.c.ref;
 462                                 disp = dseg_add_unique_address(cd, cr);
 463
 464 /*                              PROFILE_CYCLE_STOP; */
 465
 466                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
 467                                                                           cr, disp);
 468
 469 /*                              PROFILE_CYCLE_START; */
 470
 471                                 M_ALD(d, RIP, disp);
 472                         }
 473                         else {
 474                                 if (iptr->sx.val.anyptr == 0) {
 475                                         M_CLR(d);
 476                                 }
 477                                 else {
 478                                         disp = dseg_add_address(cd, iptr->sx.val.anyptr);
 479                                         M_ALD(d, RIP, disp);
 480                                 }
 481                         }
 482                         emit_store_dst(jd, iptr, d);
 483                         break;
 484
 485
 486                 /* load/store/copy/move operations ************************************/
 487
 488                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 489                 case ICMD_ALOAD:      /* s1 = local variable                          */
 490                 case ICMD_LLOAD:
 491                 case ICMD_FLOAD:
 492                 case ICMD_DLOAD:
 493                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 494                 case ICMD_LSTORE:
 495                 case ICMD_FSTORE:
 496                 case ICMD_DSTORE:
 497                 case ICMD_COPY:
 498                 case ICMD_MOVE:
 499
 500                         emit_copy(jd, iptr);
 501                         break;
 502
 503                 case ICMD_ASTORE:
 504                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 505                                 emit_copy(jd, iptr);
 506                         break;
 507
 508                 /* integer operations *************************************************/
 509
 510                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 511
 512                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 513                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 514                         M_INTMOVE(s1, d);
 515                         M_INEG(d);
 516                         emit_store_dst(jd, iptr, d);
 517                         break;
 518
 519                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 520
 521                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 522                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 523                         M_INTMOVE(s1, d);
 524                         M_LNEG(d);
 525                         emit_store_dst(jd, iptr, d);
 526                         break;
 527
 528                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 529
 530                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 531                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 532                         M_ISEXT(s1, d);
 533                         emit_store_dst(jd, iptr, d);
 534                         break;
 535
 536                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 537
 538                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 539                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 540                         M_IMOV(s1, d);
 541                         emit_store_dst(jd, iptr, d);
 542                         break;
 543
 544                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 545
 546                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 547                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 548                         M_BSEXT(s1, d);
 549                         emit_store_dst(jd, iptr, d);
 550                         break;
 551
 552                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 553
 554                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 555                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 556                         M_CZEXT(s1, d);
 557                         emit_store_dst(jd, iptr, d);
 558                         break;
 559
 560                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 561
 562                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 563                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 564                         M_SSEXT(s1, d);
 565                         emit_store_dst(jd, iptr, d);
 566                         break;
 567
 568
 569                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 570
 571                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 572                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 573                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 574                         if (s2 == d)
 575                                 M_IADD(s1, d);
 576                         else {
 577                                 M_INTMOVE(s1, d);
 578                                 M_IADD(s2, d);
 579                         }
 580                         emit_store_dst(jd, iptr, d);
 581                         break;
 582
 583                 case ICMD_IINC:
 584                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 585                                       /* sx.val.i = constant                             */
 586
 587                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 588                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 589
 590                         /* Using inc and dec is not faster than add (tested with
 591                            sieve). */
 592
 593                         M_INTMOVE(s1, d);
 594                         M_IADD_IMM(iptr->sx.val.i, d);
 595                         emit_store_dst(jd, iptr, d);
 596                         break;
 597
 598                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 599
 600                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 601                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 602                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 603                         if (s2 == d)
 604                                 M_LADD(s1, d);
 605                         else {
 606                                 M_INTMOVE(s1, d);
 607                                 M_LADD(s2, d);
 608                         }
 609                         emit_store_dst(jd, iptr, d);
 610                         break;
 611
 612                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 613                                       /* sx.val.l = constant                             */
 614
 615                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 616                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 617                         M_INTMOVE(s1, d);
 618                         if (IS_IMM32(iptr->sx.val.l))
 619                                 M_LADD_IMM(iptr->sx.val.l, d);
 620                         else {
 621                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 622                                 M_LADD(REG_ITMP2, d);
 623                         }
 624                         emit_store_dst(jd, iptr, d);
 625                         break;
 626
 627                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 628
 629                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 630                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 631                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 632                         if (s2 == d) {
 633                                 M_INTMOVE(s1, REG_ITMP1);
 634                                 M_ISUB(s2, REG_ITMP1);
 635                                 M_INTMOVE(REG_ITMP1, d);
 636                         } else {
 637                                 M_INTMOVE(s1, d);
 638                                 M_ISUB(s2, d);
 639                         }
 640                         emit_store_dst(jd, iptr, d);
 641                         break;
 642
 643                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 644                                       /* sx.val.i = constant                             */
 645
 646                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 647                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 648                         M_INTMOVE(s1, d);
 649                         M_ISUB_IMM(iptr->sx.val.i, d);
 650                         emit_store_dst(jd, iptr, d);
 651                         break;
 652
 653                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 654
 655                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 656                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 657                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 658                         if (s2 == d) {
 659                                 M_INTMOVE(s1, REG_ITMP1);
 660                                 M_LSUB(s2, REG_ITMP1);
 661                                 M_INTMOVE(REG_ITMP1, d);
 662                         } else {
 663                                 M_INTMOVE(s1, d);
 664                                 M_LSUB(s2, d);
 665                         }
 666                         emit_store_dst(jd, iptr, d);
 667                         break;
 668
 669                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 670                                       /* sx.val.l = constant                             */
 671
 672                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 673                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 674                         M_INTMOVE(s1, d);
 675                         if (IS_IMM32(iptr->sx.val.l))
 676                                 M_LSUB_IMM(iptr->sx.val.l, d);
 677                         else {
 678                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 679                                 M_LSUB(REG_ITMP2, d);
 680                         }
 681                         emit_store_dst(jd, iptr, d);
 682                         break;
 683
 684                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 685
 686                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 687                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 688                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 689                         if (s2 == d)
 690                                 M_IMUL(s1, d);
 691                         else {
 692                                 M_INTMOVE(s1, d);
 693                                 M_IMUL(s2, d);
 694                         }
 695                         emit_store_dst(jd, iptr, d);
 696                         break;
 697
 698                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 699                                       /* sx.val.i = constant                             */
 700
 701                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 702                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 703                         if (iptr->sx.val.i == 2) {
 704                                 M_INTMOVE(s1, d);
 705                                 M_ISLL_IMM(1, d);
 706                         } else
 707                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 708                         emit_store_dst(jd, iptr, d);
 709                         break;
 710
 711                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 712
 713                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 714                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 715                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 716                         if (s2 == d)
 717                                 M_LMUL(s1, d);
 718                         else {
 719                                 M_INTMOVE(s1, d);
 720                                 M_LMUL(s2, d);
 721                         }
 722                         emit_store_dst(jd, iptr, d);
 723                         break;
 724
 725                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 726                                       /* sx.val.l = constant                             */
 727
 728                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 729                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 730                         if (IS_IMM32(iptr->sx.val.l))
 731                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 732                         else {
 733                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 734                                 M_INTMOVE(s1, d);
 735                                 M_LMUL(REG_ITMP2, d);
 736                         }
 737                         emit_store_dst(jd, iptr, d);
 738                         break;
 739
 740                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 741
 742                         s1 = emit_load_s1(jd, iptr, RAX);
 743                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 744                         d = codegen_reg_of_dst(jd, iptr, RAX);
 745
 746                         M_INTMOVE(s1, RAX);
 747                         M_INTMOVE(s2, REG_ITMP3);
 748                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 749
 750                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 751
 752                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 753                         M_BNE(4 + 6);
 754                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 755                         M_BEQ(1 + 3);                                          /* 6 bytes */
 756
 757                         emit_cltd(cd);                                         /* 1 byte  */
 758                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 759
 760                         M_INTMOVE(RAX, d);
 761                         emit_store_dst(jd, iptr, d);
 762                         dst = VAROP(iptr->dst);
 763                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 764                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 765                         break;
 766
 767                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 768
 769                         s1 = emit_load_s1(jd, iptr, RAX);
 770                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 771                         d = codegen_reg_of_dst(jd, iptr, RDX);
 772
 773                         M_INTMOVE(s1, RAX);
 774                         M_INTMOVE(s2, REG_ITMP3);
 775                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 776
 777                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 778
 779                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 780                         M_BNE(3 + 4 + 6);
 781                         M_CLR(RDX);                                            /* 3 bytes */
 782                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 783                         M_BEQ(1 + 3);                                          /* 6 bytes */
 784
 785                         emit_cltd(cd);                                         /* 1 byte  */
 786                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 787
 788                         M_INTMOVE(RDX, d);
 789                         emit_store_dst(jd, iptr, d);
 790                         dst = VAROP(iptr->dst);
 791                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 792                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 793                         break;
 794
 795                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 796                                       /* sx.val.i = constant                             */
 797
 798                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 799                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 800                         M_INTMOVE(s1, REG_ITMP1);
 801                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 802                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 803                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 804                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 805                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 806                         emit_store_dst(jd, iptr, d);
 807                         break;
 808
 809                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 810                                       /* sx.val.i = constant                             */
 811
 812                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 813                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 814                         M_INTMOVE(s1, REG_ITMP1);
 815                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 816                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 817                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 818                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 819                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 820                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 821                         emit_store_dst(jd, iptr, d);
 822                         break;
 823
 824
 825                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 826
 827                         s1 = emit_load_s1(jd, iptr, RAX);
 828                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 829                         d = codegen_reg_of_dst(jd, iptr, RAX);
 830
 831                         M_INTMOVE(s1, RAX);
 832                         M_INTMOVE(s2, REG_ITMP3);
 833                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 834
 835                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 836
 837                         /* check as described in jvm spec */
 838                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 839                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 840                         M_BNE(4 + 6);
 841                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 842                         M_BEQ(2 + 3);                                          /* 6 bytes */
 843
 844                         emit_cqto(cd);                                         /* 2 bytes */
 845                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 846
 847                         M_INTMOVE(RAX, d);
 848                         emit_store_dst(jd, iptr, d);
 849                         dst = VAROP(iptr->dst);
 850                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 851                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 852                         break;
 853
 854                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 855
 856                         s1 = emit_load_s1(jd, iptr, RAX);
 857                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 858                         d = codegen_reg_of_dst(jd, iptr, RDX);
 859
 860                         M_INTMOVE(s1, RAX);
 861                         M_INTMOVE(s2, REG_ITMP3);
 862                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 863
 864                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 865
 866                         /* check as described in jvm spec */
 867                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 868                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 869                         M_BNE(3 + 4 + 6);
 870                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 871                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 872                         M_BEQ(2 + 3);                                          /* 6 bytes */
 873
 874                         emit_cqto(cd);                                         /* 2 bytes */
 875                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 876
 877                         M_INTMOVE(RDX, d);
 878                         emit_store_dst(jd, iptr, d);
 879                         dst = VAROP(iptr->dst);
 880                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 881                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 882                         break;
 883
 884                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 885                                       /* sx.val.i = constant                             */
 886
 887                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 888                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 889                         M_INTMOVE(s1, REG_ITMP1);
 890                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 891                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 892                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 893                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 894                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 895                         emit_store_dst(jd, iptr, d);
 896                         break;
 897
 898                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 899                                       /* sx.val.l = constant                             */
 900
 901                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 902                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 903                         M_INTMOVE(s1, REG_ITMP1);
 904                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 905                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 906                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 907                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 908                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 909                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 910                         emit_store_dst(jd, iptr, d);
 911                         break;
 912
 913                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 914
 915                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 916                         emit_ishift(jd, SHIFT_SHL, iptr);
 917                         break;
 918
 919                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 920                                       /* sx.val.i = constant                             */
 921
 922                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 923                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 924                         M_INTMOVE(s1, d);
 925                         M_ISLL_IMM(iptr->sx.val.i, d);
 926                         emit_store_dst(jd, iptr, d);
 927                         break;
 928
 929                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 930
 931                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 932                         emit_ishift(jd, SHIFT_SAR, iptr);
 933                         break;
 934
 935                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 936                                       /* sx.val.i = constant                             */
 937
 938                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 939                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 940                         M_INTMOVE(s1, d);
 941                         M_ISRA_IMM(iptr->sx.val.i, d);
 942                         emit_store_dst(jd, iptr, d);
 943                         break;
 944
 945                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 946
 947                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 948                         emit_ishift(jd, SHIFT_SHR, iptr);
 949                         break;
 950
 951                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 952                                       /* sx.val.i = constant                             */
 953
 954                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 955                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 956                         M_INTMOVE(s1, d);
 957                         M_ISRL_IMM(iptr->sx.val.i, d);
 958                         emit_store_dst(jd, iptr, d);
 959                         break;
 960
 961                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 962
 963                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 964                         emit_lshift(jd, SHIFT_SHL, iptr);
 965                         break;
 966
 967         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 968                                           /* sx.val.i = constant                             */
 969
 970                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 971                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 972                         M_INTMOVE(s1, d);
 973                         M_LSLL_IMM(iptr->sx.val.i, d);
 974                         emit_store_dst(jd, iptr, d);
 975                         break;
 976
 977                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 978
 979                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 980                         emit_lshift(jd, SHIFT_SAR, iptr);
 981                         break;
 982
 983                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 984                                       /* sx.val.i = constant                             */
 985
 986                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 987                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 988                         M_INTMOVE(s1, d);
 989                         M_LSRA_IMM(iptr->sx.val.i, d);
 990                         emit_store_dst(jd, iptr, d);
 991                         break;
 992
 993                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 994
 995                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 996                         emit_lshift(jd, SHIFT_SHR, iptr);
 997                         break;
 998
 999                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1000                                       /* sx.val.l = constant                             */
1001
1002                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1003                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1004                         M_INTMOVE(s1, d);
1005                         M_LSRL_IMM(iptr->sx.val.i, d);
1006                         emit_store_dst(jd, iptr, d);
1007                         break;
1008
1009                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1010
1011                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1012                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1013                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1014                         if (s2 == d)
1015                                 M_IAND(s1, d);
1016                         else {
1017                                 M_INTMOVE(s1, d);
1018                                 M_IAND(s2, d);
1019                         }
1020                         emit_store_dst(jd, iptr, d);
1021                         break;
1022
1023                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1024                                       /* sx.val.i = constant                             */
1025
1026                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1027                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1028                         M_INTMOVE(s1, d);
1029                         M_IAND_IMM(iptr->sx.val.i, d);
1030                         emit_store_dst(jd, iptr, d);
1031                         break;
1032
1033                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1034
1035                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1036                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1037                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1038                         if (s2 == d)
1039                                 M_LAND(s1, d);
1040                         else {
1041                                 M_INTMOVE(s1, d);
1042                                 M_LAND(s2, d);
1043                         }
1044                         emit_store_dst(jd, iptr, d);
1045                         break;
1046
1047                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1048                                       /* sx.val.l = constant                             */
1049
1050                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1051                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1052                         M_INTMOVE(s1, d);
1053                         if (IS_IMM32(iptr->sx.val.l))
1054                                 M_LAND_IMM(iptr->sx.val.l, d);
1055                         else {
1056                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1057                                 M_LAND(REG_ITMP2, d);
1058                         }
1059                         emit_store_dst(jd, iptr, d);
1060                         break;
1061
1062                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1063
1064                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1065                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1066                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1067                         if (s2 == d)
1068                                 M_IOR(s1, d);
1069                         else {
1070                                 M_INTMOVE(s1, d);
1071                                 M_IOR(s2, d);
1072                         }
1073                         emit_store_dst(jd, iptr, d);
1074                         break;
1075
1076                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1077                                       /* sx.val.i = constant                             */
1078
1079                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1080                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1081                         M_INTMOVE(s1, d);
1082                         M_IOR_IMM(iptr->sx.val.i, d);
1083                         emit_store_dst(jd, iptr, d);
1084                         break;
1085
1086                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1087
1088                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1089                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1090                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1091                         if (s2 == d)
1092                                 M_LOR(s1, d);
1093                         else {
1094                                 M_INTMOVE(s1, d);
1095                                 M_LOR(s2, d);
1096                         }
1097                         emit_store_dst(jd, iptr, d);
1098                         break;
1099
1100                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1101                                       /* sx.val.l = constant                             */
1102
1103                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1104                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1105                         M_INTMOVE(s1, d);
1106                         if (IS_IMM32(iptr->sx.val.l))
1107                                 M_LOR_IMM(iptr->sx.val.l, d);
1108                         else {
1109                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1110                                 M_LOR(REG_ITMP2, d);
1111                         }
1112                         emit_store_dst(jd, iptr, d);
1113                         break;
1114
1115                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1116
1117                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1118                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1119                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1120                         if (s2 == d)
1121                                 M_IXOR(s1, d);
1122                         else {
1123                                 M_INTMOVE(s1, d);
1124                                 M_IXOR(s2, d);
1125                         }
1126                         emit_store_dst(jd, iptr, d);
1127                         break;
1128
1129                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1130                                       /* sx.val.i = constant                             */
1131
1132                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1133                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1134                         M_INTMOVE(s1, d);
1135                         M_IXOR_IMM(iptr->sx.val.i, d);
1136                         emit_store_dst(jd, iptr, d);
1137                         break;
1138
1139                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1140
1141                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1142                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1143                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1144                         if (s2 == d)
1145                                 M_LXOR(s1, d);
1146                         else {
1147                                 M_INTMOVE(s1, d);
1148                                 M_LXOR(s2, d);
1149                         }
1150                         emit_store_dst(jd, iptr, d);
1151                         break;
1152
1153                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1154                                       /* sx.val.l = constant                             */
1155
1156                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1157                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1158                         M_INTMOVE(s1, d);
1159                         if (IS_IMM32(iptr->sx.val.l))
1160                                 M_LXOR_IMM(iptr->sx.val.l, d);
1161                         else {
1162                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1163                                 M_LXOR(REG_ITMP2, d);
1164                         }
1165                         emit_store_dst(jd, iptr, d);
1166                         break;
1167
1168
1169                 /* floating operations ************************************************/
1170
1171                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1172
1173                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1174                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1175                         disp = dseg_add_s4(cd, 0x80000000);
1176                         M_FLTMOVE(s1, d);
1177                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1178                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1179                         emit_store_dst(jd, iptr, d);
1180                         break;
1181
1182                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1183
1184                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1185                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1186                         disp = dseg_add_s8(cd, 0x8000000000000000);
1187                         M_FLTMOVE(s1, d);
1188                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1189                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1190                         emit_store_dst(jd, iptr, d);
1191                         break;
1192
1193                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1194
1195                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1196                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1197                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1198                         if (s2 == d)
1199                                 M_FADD(s1, d);
1200                         else {
1201                                 M_FLTMOVE(s1, d);
1202                                 M_FADD(s2, d);
1203                         }
1204                         emit_store_dst(jd, iptr, d);
1205                         break;
1206
1207                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1208
1209                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1210                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1211                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1212                         if (s2 == d)
1213                                 M_DADD(s1, d);
1214                         else {
1215                                 M_FLTMOVE(s1, d);
1216                                 M_DADD(s2, d);
1217                         }
1218                         emit_store_dst(jd, iptr, d);
1219                         break;
1220
1221                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1222
1223                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1224                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1225                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1226                         if (s2 == d) {
1227                                 M_FLTMOVE(s2, REG_FTMP2);
1228                                 s2 = REG_FTMP2;
1229                         }
1230                         M_FLTMOVE(s1, d);
1231                         M_FSUB(s2, d);
1232                         emit_store_dst(jd, iptr, d);
1233                         break;
1234
1235                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1236
1237                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1238                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1239                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1240                         if (s2 == d) {
1241                                 M_FLTMOVE(s2, REG_FTMP2);
1242                                 s2 = REG_FTMP2;
1243                         }
1244                         M_FLTMOVE(s1, d);
1245                         M_DSUB(s2, d);
1246                         emit_store_dst(jd, iptr, d);
1247                         break;
1248
1249                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1250
1251                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1252                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1253                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1254                         if (s2 == d)
1255                                 M_FMUL(s1, d);
1256                         else {
1257                                 M_FLTMOVE(s1, d);
1258                                 M_FMUL(s2, d);
1259                         }
1260                         emit_store_dst(jd, iptr, d);
1261                         break;
1262
1263                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1264
1265                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1266                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1267                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1268                         if (s2 == d)
1269                                 M_DMUL(s1, d);
1270                         else {
1271                                 M_FLTMOVE(s1, d);
1272                                 M_DMUL(s2, d);
1273                         }
1274                         emit_store_dst(jd, iptr, d);
1275                         break;
1276
1277                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1278
1279                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1280                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1281                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1282                         if (s2 == d) {
1283                                 M_FLTMOVE(s2, REG_FTMP2);
1284                                 s2 = REG_FTMP2;
1285                         }
1286                         M_FLTMOVE(s1, d);
1287                         M_FDIV(s2, d);
1288                         emit_store_dst(jd, iptr, d);
1289                         break;
1290
1291                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1292
1293                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1294                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1295                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1296                         if (s2 == d) {
1297                                 M_FLTMOVE(s2, REG_FTMP2);
1298                                 s2 = REG_FTMP2;
1299                         }
1300                         M_FLTMOVE(s1, d);
1301                         M_DDIV(s2, d);
1302                         emit_store_dst(jd, iptr, d);
1303                         break;
1304
1305                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1306
1307                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1308                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1309                         M_CVTIF(s1, d);
1310                         emit_store_dst(jd, iptr, d);
1311                         break;
1312
1313                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1314
1315                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1316                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1317                         M_CVTID(s1, d);
1318                         emit_store_dst(jd, iptr, d);
1319                         break;
1320
1321                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1322
1323                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1324                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1325                         M_CVTLF(s1, d);
1326                         emit_store_dst(jd, iptr, d);
1327                         break;
1328
1329                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1330
1331                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1332                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1333                         M_CVTLD(s1, d);
1334                         emit_store_dst(jd, iptr, d);
1335                         break;
1336
1337                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1338
1339                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1340                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1341                         M_CVTFI(s1, d);
1342                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1343                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1344                                 ((REG_RESULT == d) ? 0 : 3);
1345                         M_BNE(disp);
1346                         M_FLTMOVE(s1, REG_FTMP1);
1347                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1348                         M_CALL(REG_ITMP2);
1349                         M_INTMOVE(REG_RESULT, d);
1350                         emit_store_dst(jd, iptr, d);
1351                         break;
1352
1353                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1354
1355                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1356                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1357                         M_CVTDI(s1, d);
1358                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1359                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1360                                 ((REG_RESULT == d) ? 0 : 3);
1361                         M_BNE(disp);
1362                         M_FLTMOVE(s1, REG_FTMP1);
1363                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1364                         M_CALL(REG_ITMP2);
1365                         M_INTMOVE(REG_RESULT, d);
1366                         emit_store_dst(jd, iptr, d);
1367                         break;
1368
1369                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1370
1371                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1372                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1373                         M_CVTFL(s1, d);
1374                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1375                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1376                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1377                                 ((REG_RESULT == d) ? 0 : 3);
1378                         M_BNE(disp);
1379                         M_FLTMOVE(s1, REG_FTMP1);
1380                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1381                         M_CALL(REG_ITMP2);
1382                         M_INTMOVE(REG_RESULT, d);
1383                         emit_store_dst(jd, iptr, d);
1384                         break;
1385
1386                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1387
1388                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1389                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1390                         M_CVTDL(s1, d);
1391                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1392                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1393                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1394                                 ((REG_RESULT == d) ? 0 : 3);
1395                         M_BNE(disp);
1396                         M_FLTMOVE(s1, REG_FTMP1);
1397                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1398                         M_CALL(REG_ITMP2);
1399                         M_INTMOVE(REG_RESULT, d);
1400                         emit_store_dst(jd, iptr, d);
1401                         break;
1402
1403                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1404
1405                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1406                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1407                         M_CVTFD(s1, d);
1408                         emit_store_dst(jd, iptr, d);
1409                         break;
1410
1411                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1412
1413                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1414                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1415                         M_CVTDF(s1, d);
1416                         emit_store_dst(jd, iptr, d);
1417                         break;
1418
1419                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1420                                           /* == => 0, < => 1, > => -1 */
1421
1422                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1423                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1424                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1425                         M_CLR(d);
1426                         M_MOV_IMM(1, REG_ITMP1);
1427                         M_MOV_IMM(-1, REG_ITMP2);
1428                         emit_ucomiss_reg_reg(cd, s1, s2);
1429                         M_CMOVULT(REG_ITMP1, d);
1430                         M_CMOVUGT(REG_ITMP2, d);
1431                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1432                         emit_store_dst(jd, iptr, d);
1433                         break;
1434
1435                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1436                                           /* == => 0, < => 1, > => -1 */
1437
1438                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1439                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1440                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1441                         M_CLR(d);
1442                         M_MOV_IMM(1, REG_ITMP1);
1443                         M_MOV_IMM(-1, REG_ITMP2);
1444                         emit_ucomiss_reg_reg(cd, s1, s2);
1445                         M_CMOVULT(REG_ITMP1, d);
1446                         M_CMOVUGT(REG_ITMP2, d);
1447                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1448                         emit_store_dst(jd, iptr, d);
1449                         break;
1450
1451                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1452                                           /* == => 0, < => 1, > => -1 */
1453
1454                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1455                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1456                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1457                         M_CLR(d);
1458                         M_MOV_IMM(1, REG_ITMP1);
1459                         M_MOV_IMM(-1, REG_ITMP2);
1460                         emit_ucomisd_reg_reg(cd, s1, s2);
1461                         M_CMOVULT(REG_ITMP1, d);
1462                         M_CMOVUGT(REG_ITMP2, d);
1463                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1464                         emit_store_dst(jd, iptr, d);
1465                         break;
1466
1467                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1468                                           /* == => 0, < => 1, > => -1 */
1469
1470                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1471                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1472                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1473                         M_CLR(d);
1474                         M_MOV_IMM(1, REG_ITMP1);
1475                         M_MOV_IMM(-1, REG_ITMP2);
1476                         emit_ucomisd_reg_reg(cd, s1, s2);
1477                         M_CMOVULT(REG_ITMP1, d);
1478                         M_CMOVUGT(REG_ITMP2, d);
1479                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1480                         emit_store_dst(jd, iptr, d);
1481                         break;
1482
1483
1484                 /* memory operations **************************************************/
1485
1486                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1487
1488                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1489                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1490                         /* implicit null-pointer check */
1491                         M_ILD(d, s1, OFFSET(java_array_t, size));
1492                         emit_store_dst(jd, iptr, d);
1493                         break;
1494
1495                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1496
1497                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1498                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1499                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1500                         /* implicit null-pointer check */
1501                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1502                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1503                         emit_store_dst(jd, iptr, d);
1504                         break;
1505
1506                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1507
1508                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1509                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1510                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1511                         /* implicit null-pointer check */
1512                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1513                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1514                         emit_store_dst(jd, iptr, d);
1515                         break;
1516
1517                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1518
1519                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1520                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1521                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1522                         /* implicit null-pointer check */
1523                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1524                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1525                         emit_store_dst(jd, iptr, d);
1526                         break;
1527
1528                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1529
1530                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1531                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1532                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1533                         /* implicit null-pointer check */
1534                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1535                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1536                         emit_store_dst(jd, iptr, d);
1537                         break;
1538
1539                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1540
1541                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1542                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1543                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1544                         /* implicit null-pointer check */
1545                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1546                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1547                         emit_store_dst(jd, iptr, d);
1548                         break;
1549
1550                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1551
1552                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1553                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1554                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1555                         /* implicit null-pointer check */
1556                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1557                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1558                         emit_store_dst(jd, iptr, d);
1559                         break;
1560
1561                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1562
1563                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1564                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1565                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1566                         /* implicit null-pointer check */
1567                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1568                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1569                         emit_store_dst(jd, iptr, d);
1570                         break;
1571
1572                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1573
1574                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1575                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1576                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1577                         /* implicit null-pointer check */
1578                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1579                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1580                         emit_store_dst(jd, iptr, d);
1581                         break;
1582
1583
1584                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1585
1586                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1587                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1588                         /* implicit null-pointer check */
1589                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1590                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1591                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1592                         break;
1593
1594                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1595
1596                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1597                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1598                         /* implicit null-pointer check */
1599                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1600                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1601                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1602                         break;
1603
1604                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1605
1606                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1607                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1608                         /* implicit null-pointer check */
1609                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1610                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1611                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1612                         break;
1613
1614                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1615
1616                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1617                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1618                         /* implicit null-pointer check */
1619                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1620                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1621                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1622                         break;
1623
1624                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1625
1626                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1627                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1628                         /* implicit null-pointer check */
1629                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1630                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1631                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1632                         break;
1633
1634                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1635
1636                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1637                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1638                         /* implicit null-pointer check */
1639                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1640                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1641                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1642                         break;
1643
1644                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1645
1646                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1647                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1648                         /* implicit null-pointer check */
1649                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1650                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1651                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1652                         break;
1653
1654                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1655
1656                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1657                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1658                         /* implicit null-pointer check */
1659                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1660                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1661
1662                         M_MOV(s1, REG_A0);
1663                         M_MOV(s3, REG_A1);
1664                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
1665                         M_CALL(REG_ITMP1);
1666                         emit_arraystore_check(cd, iptr);
1667
1668                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1669                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1670                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1671                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1672                         break;
1673
1674
1675                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1676
1677                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1678                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1679                         /* implicit null-pointer check */
1680                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1681                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1682                         break;
1683
1684                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1685
1686                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1687                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1688                         /* implicit null-pointer check */
1689                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1690                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1691                         break;
1692
1693                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1694
1695                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1696                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1697                         /* implicit null-pointer check */
1698                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1699                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1700                         break;
1701
1702                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1703
1704                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1705                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1706                         /* implicit null-pointer check */
1707                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1708                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1709                         break;
1710
1711                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1712
1713                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1714                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1715                         /* implicit null-pointer check */
1716                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1717
1718                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1719                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1720                         }
1721                         else {
1722                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1723                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1724                         }
1725                         break;
1726
1727                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1728
1729                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1730                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1731                         /* implicit null-pointer check */
1732                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1733                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1734                         break;
1735
1736
1737                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1738
1739                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1740                                 uf        = iptr->sx.s23.s3.uf;
1741                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1742                                 disp      = dseg_add_unique_address(cd, uf);
1743
1744 /*                              PROFILE_CYCLE_STOP; */
1745
1746                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1747
1748 /*                              PROFILE_CYCLE_START; */
1749                         }
1750                         else {
1751                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1752                                 fieldtype = fi->type;
1753                                 disp      = dseg_add_address(cd, fi->value);
1754
1755                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1756                                         PROFILE_CYCLE_STOP;
1757
1758                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1759                                                                                   fi->clazz, 0);
1760
1761                                         PROFILE_CYCLE_START;
1762                                 }
1763                         }
1764
1765                         /* This approach is much faster than moving the field
1766                            address inline into a register. */
1767
1768                         M_ALD(REG_ITMP1, RIP, disp);
1769
1770                         switch (fieldtype) {
1771                         case TYPE_INT:
1772                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1773                                 M_ILD(d, REG_ITMP1, 0);
1774                                 break;
1775                         case TYPE_LNG:
1776                         case TYPE_ADR:
1777                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1778                                 M_LLD(d, REG_ITMP1, 0);
1779                                 break;
1780                         case TYPE_FLT:
1781                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1782                                 M_FLD(d, REG_ITMP1, 0);
1783                                 break;
1784                         case TYPE_DBL:
1785                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1786                                 M_DLD(d, REG_ITMP1, 0);
1787                                 break;
1788                         }
1789                         emit_store_dst(jd, iptr, d);
1790                         break;
1791
1792                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1793
1794                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1795                                 uf        = iptr->sx.s23.s3.uf;
1796                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1797                                 disp      = dseg_add_unique_address(cd, uf);
1798
1799 /*                              PROFILE_CYCLE_STOP; */
1800
1801                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1802
1803 /*                              PROFILE_CYCLE_START; */
1804                         }
1805                         else {
1806                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1807                                 fieldtype = fi->type;
1808                                 disp      = dseg_add_address(cd, fi->value);
1809
1810                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1811                                         PROFILE_CYCLE_STOP;
1812
1813                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1814                                                                                   fi->clazz, 0);
1815
1816                                         PROFILE_CYCLE_START;
1817                                 }
1818                         }
1819
1820                         /* This approach is much faster than moving the field
1821                            address inline into a register. */
1822
1823                         M_ALD(REG_ITMP1, RIP, disp);
1824
1825                         switch (fieldtype) {
1826                         case TYPE_INT:
1827                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1828                                 M_IST(s1, REG_ITMP1, 0);
1829                                 break;
1830                         case TYPE_LNG:
1831                         case TYPE_ADR:
1832                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1833                                 M_LST(s1, REG_ITMP1, 0);
1834                                 break;
1835                         case TYPE_FLT:
1836                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1837                                 M_FST(s1, REG_ITMP1, 0);
1838                                 break;
1839                         case TYPE_DBL:
1840                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1841                                 M_DST(s1, REG_ITMP1, 0);
1842                                 break;
1843                         }
1844                         break;
1845
1846                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1847                                           /* val = value (in current instruction)     */
1848                                           /* following NOP)                           */
1849
1850                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1851                                 uf        = iptr->sx.s23.s3.uf;
1852                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1853                                 disp      = dseg_add_unique_address(cd, uf);
1854
1855 /*                              PROFILE_CYCLE_STOP; */
1856
1857                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1858
1859 /*                              PROFILE_CYCLE_START; */
1860                         }
1861                         else {
1862                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1863                                 fieldtype = fi->type;
1864                                 disp      = dseg_add_address(cd, fi->value);
1865
1866                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1867                                         PROFILE_CYCLE_STOP;
1868
1869                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1870                                                                                   fi->clazz, 0);
1871
1872                                         PROFILE_CYCLE_START;
1873                                 }
1874                         }
1875
1876                         /* This approach is much faster than moving the field
1877                            address inline into a register. */
1878
1879                         M_ALD(REG_ITMP1, RIP, disp);
1880
1881                         switch (fieldtype) {
1882                         case TYPE_INT:
1883                         case TYPE_FLT:
1884                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1885                                 break;
1886                         case TYPE_LNG:
1887                         case TYPE_ADR:
1888                         case TYPE_DBL:
1889                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1890                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1891                                 else {
1892                                         M_MOV_IMM(iptr->sx.s23.s2.constval, REG_ITMP2);
1893                                         M_LST(REG_ITMP2, REG_ITMP1, 0);
1894                                 }
1895                                 break;
1896                         }
1897                         break;
1898
1899                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1900
1901                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1902
1903                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1904                                 uf        = iptr->sx.s23.s3.uf;
1905                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1906                                 disp      = 0;
1907
1908 /*                              PROFILE_CYCLE_STOP; */
1909
1910                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1911
1912 /*                              PROFILE_CYCLE_START; */
1913                         }
1914                         else {
1915                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1916                                 fieldtype = fi->type;
1917                                 disp      = fi->offset;
1918                         }
1919
1920                         /* implicit null-pointer check */
1921                         switch (fieldtype) {
1922                         case TYPE_INT:
1923                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1924                                 M_ILD32(d, s1, disp);
1925                                 break;
1926                         case TYPE_LNG:
1927                         case TYPE_ADR:
1928                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1929                                 M_LLD32(d, s1, disp);
1930                                 break;
1931                         case TYPE_FLT:
1932                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1933                                 M_FLD32(d, s1, disp);
1934                                 break;
1935                         case TYPE_DBL:
1936                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1937                                 M_DLD32(d, s1, disp);
1938                                 break;
1939                         }
1940                         emit_store_dst(jd, iptr, d);
1941                         break;
1942
1943                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1944
1945                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1946                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1947
1948                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1949                                 uf        = iptr->sx.s23.s3.uf;
1950                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1951                                 disp      = 0;
1952
1953 /*                              PROFILE_CYCLE_STOP; */
1954
1955                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1956
1957 /*                              PROFILE_CYCLE_START; */
1958                         }
1959                         else {
1960                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1961                                 fieldtype = fi->type;
1962                                 disp      = fi->offset;
1963                         }
1964
1965                         /* implicit null-pointer check */
1966                         switch (fieldtype) {
1967                         case TYPE_INT:
1968                                 M_IST32(s2, s1, disp);
1969                                 break;
1970                         case TYPE_LNG:
1971                         case TYPE_ADR:
1972                                 M_LST32(s2, s1, disp);
1973                                 break;
1974                         case TYPE_FLT:
1975                                 M_FST32(s2, s1, disp);
1976                                 break;
1977                         case TYPE_DBL:
1978                                 M_DST32(s2, s1, disp);
1979                                 break;
1980                         }
1981                         break;
1982
1983                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
1984                                           /* val = value (in current instruction)     */
1985                                           /* following NOP)                           */
1986
1987                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1988
1989                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1990                                 uf        = iptr->sx.s23.s3.uf;
1991                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1992                                 disp      = 0;
1993
1994 /*                              PROFILE_CYCLE_STOP; */
1995
1996                                 patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
1997
1998 /*                              PROFILE_CYCLE_START; */
1999                         }
2000                         else {
2001                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2002                                 fieldtype = fi->type;
2003                                 disp      = fi->offset;
2004                         }
2005
2006                         /* implicit null-pointer check */
2007                         switch (fieldtype) {
2008                         case TYPE_INT:
2009                         case TYPE_FLT:
2010                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2011                                 break;
2012                         case TYPE_LNG:
2013                         case TYPE_ADR:
2014                         case TYPE_DBL:
2015                                 /* XXX why no check for IS_IMM32? -- probably because of the patcher */
2016                                 M_MOV_IMM(iptr->sx.s23.s2.constval, REG_ITMP2);
2017                                 if (disp)  /* resolved, disp can never be 0 */
2018                                         M_LST(REG_ITMP2, s1, disp);
2019                                 else       /* unresolved */
2020                                         M_LST32(REG_ITMP2, s1, disp);
2021                                 break;
2022                         }
2023                         break;
2024
2025
2026                 /* branch operations **************************************************/
2027
2028                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2029
2030                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2031                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2032
2033                         PROFILE_CYCLE_STOP;
2034
2035 #ifdef ENABLE_VERIFIER
2036                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2037                                 uc = iptr->sx.s23.s2.uc;
2038
2039                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2040                         }
2041 #endif /* ENABLE_VERIFIER */
2042
2043                         M_CALL_IMM(0);                            /* passing exception pc */
2044                         M_POP(REG_ITMP2_XPC);
2045
2046                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2047                         M_JMP(REG_ITMP3);
2048                         break;
2049
2050                 case ICMD_GOTO:         /* ... ==> ...                                */
2051                 case ICMD_RET:
2052
2053                         emit_br(cd, iptr->dst.block);
2054                         ALIGNCODENOP;
2055                         break;
2056
2057                 case ICMD_JSR:          /* ... ==> ...                                */
2058
2059                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2060                         ALIGNCODENOP;
2061                         break;
2062
2063                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2064                 case ICMD_IFNONNULL:
2065
2066                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2067                         M_TEST(s1);
2068                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2069                         break;
2070
2071                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2072                 case ICMD_IFLT:
2073                 case ICMD_IFLE:
2074                 case ICMD_IFNE:
2075                 case ICMD_IFGT:
2076                 case ICMD_IFGE:
2077
2078                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2079                         M_ICMP_IMM(iptr->sx.val.i, s1);
2080                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2081                         break;
2082
2083                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2084                 case ICMD_IF_LNE:
2085                 case ICMD_IF_LLT:
2086                 case ICMD_IF_LGE:
2087                 case ICMD_IF_LGT:
2088                 case ICMD_IF_LLE:
2089
2090                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2091                         if (IS_IMM32(iptr->sx.val.l))
2092                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2093                         else {
2094                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2095                                 M_LCMP(REG_ITMP2, s1);
2096                         }
2097                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2098                         break;
2099
2100                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2101                 case ICMD_IF_ICMPNE:
2102                 case ICMD_IF_ICMPLT:
2103                 case ICMD_IF_ICMPGE:
2104                 case ICMD_IF_ICMPGT:
2105                 case ICMD_IF_ICMPLE:
2106
2107                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2108                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2109                         M_ICMP(s2, s1);
2110                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2111                         break;
2112
2113                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2114                 case ICMD_IF_ACMPNE:
2115
2116                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2117                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2118                         M_LCMP(s2, s1);
2119                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2120                         break;
2121
2122                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2123                 case ICMD_IF_LCMPNE:
2124                 case ICMD_IF_LCMPLT:
2125                 case ICMD_IF_LCMPGE:
2126                 case ICMD_IF_LCMPGT:
2127                 case ICMD_IF_LCMPLE:
2128
2129                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2130                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2131                         M_LCMP(s2, s1);
2132                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2133                         break;
2134
2135                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2136                 case ICMD_LRETURN:
2137
2138                         REPLACEMENT_POINT_RETURN(cd, iptr);
2139                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2140                         M_INTMOVE(s1, REG_RESULT);
2141                         goto nowperformreturn;
2142
2143                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2144
2145                         REPLACEMENT_POINT_RETURN(cd, iptr);
2146                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2147                         M_INTMOVE(s1, REG_RESULT);
2148
2149 #ifdef ENABLE_VERIFIER
2150                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2151                                 uc = iptr->sx.s23.s2.uc;
2152
2153                                 PROFILE_CYCLE_STOP;
2154
2155                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2156
2157                                 PROFILE_CYCLE_START;
2158                         }
2159 #endif /* ENABLE_VERIFIER */
2160                         goto nowperformreturn;
2161
2162                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2163                 case ICMD_DRETURN:
2164
2165                         REPLACEMENT_POINT_RETURN(cd, iptr);
2166                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2167                         M_FLTMOVE(s1, REG_FRESULT);
2168                         goto nowperformreturn;
2169
2170                 case ICMD_RETURN:      /* ...  ==> ...                                */
2171
2172                         REPLACEMENT_POINT_RETURN(cd, iptr);
2173
2174 nowperformreturn:
2175                         {
2176                         s4 i, p;
2177
2178                         p = cd->stackframesize;
2179
2180 #if !defined(NDEBUG)
2181                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2182                                 emit_verbosecall_exit(jd);
2183 #endif /* !defined(NDEBUG) */
2184
2185 #if defined(ENABLE_THREADS)
2186                         if (checksync && code_is_synchronized(code)) {
2187                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2188
2189                                 /* we need to save the proper return value */
2190                                 switch (iptr->opc) {
2191                                 case ICMD_IRETURN:
2192                                 case ICMD_ARETURN:
2193                                 case ICMD_LRETURN:
2194                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2195                                         break;
2196                                 case ICMD_FRETURN:
2197                                 case ICMD_DRETURN:
2198                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2199                                         break;
2200                                 }
2201
2202                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2203                                 M_CALL(REG_ITMP1);
2204
2205                                 /* and now restore the proper return value */
2206                                 switch (iptr->opc) {
2207                                 case ICMD_IRETURN:
2208                                 case ICMD_ARETURN:
2209                                 case ICMD_LRETURN:
2210                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2211                                         break;
2212                                 case ICMD_FRETURN:
2213                                 case ICMD_DRETURN:
2214                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2215                                         break;
2216                                 }
2217                         }
2218 #endif
2219
2220                         /* restore saved registers */
2221
2222                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2223                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2224                         }
2225                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2226                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2227                         }
2228
2229                         /* deallocate stack */
2230
2231                         if (cd->stackframesize)
2232                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2233
2234                         /* generate method profiling code */
2235
2236                         PROFILE_CYCLE_STOP;
2237
2238                         M_RET;
2239                         }
2240                         break;
2241
2242
2243                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2244                         {
2245                                 s4 i, l;
2246                                 branch_target_t *table;
2247
2248                                 table = iptr->dst.table;
2249
2250                                 l = iptr->sx.s23.s2.tablelow;
2251                                 i = iptr->sx.s23.s3.tablehigh;
2252
2253                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2254                                 M_INTMOVE(s1, REG_ITMP1);
2255
2256                                 if (l != 0)
2257                                         M_ISUB_IMM(l, REG_ITMP1);
2258
2259                                 /* number of targets */
2260                                 i = i - l + 1;
2261
2262                 /* range check */
2263
2264                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2265                                 emit_bugt(cd, table[0].block);
2266
2267                                 /* build jump table top down and use address of lowest entry */
2268
2269                                 table += i;
2270
2271                                 while (--i >= 0) {
2272                                         dseg_add_target(cd, table->block);
2273                                         --table;
2274                                 }
2275
2276                                 /* length of dataseg after last dseg_add_target is used
2277                                    by load */
2278
2279                                 M_MOV_IMM(0, REG_ITMP2);
2280                                 dseg_adddata(cd);
2281                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2282                                 M_JMP(REG_ITMP1);
2283                         }
2284                         break;
2285
2286
2287                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2288                         {
2289                                 s4 i;
2290                                 lookup_target_t *lookup;
2291
2292                                 lookup = iptr->dst.lookup;
2293
2294                                 i = iptr->sx.s23.s2.lookupcount;
2295
2296                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2297                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2298
2299                                 while (--i >= 0) {
2300                                         M_ICMP_IMM(lookup->value, s1);
2301                                         emit_beq(cd, lookup->target.block);
2302                                         lookup++;
2303                                 }
2304
2305                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2306                                 ALIGNCODENOP;
2307                         }
2308                         break;
2309
2310
2311                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2312
2313                         REPLACEMENT_POINT_FORGC_BUILTIN(cd, iptr);
2314
2315                         bte = iptr->sx.s23.s3.bte;
2316                         md  = bte->md;
2317                         goto gen_method;
2318
2319                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2320
2321                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2322                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2323                 case ICMD_INVOKEINTERFACE:
2324
2325                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2326
2327                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2328                                 lm = NULL;
2329                                 um = iptr->sx.s23.s3.um;
2330                                 md = um->methodref->parseddesc.md;
2331                         }
2332                         else {
2333                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2334                                 um = NULL;
2335                                 md = lm->parseddesc;
2336                         }
2337
2338 gen_method:
2339                         s3 = md->paramcount;
2340
2341                         MCODECHECK((20 * s3) + 128);
2342
2343                         /* copy arguments to registers or stack location */
2344
2345                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2346                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2347                                 d   = md->params[s3].regoff;
2348
2349                                 /* already preallocated (ARGVAR)? */
2350
2351                                 if (var->flags & PREALLOC)
2352                                         continue;
2353
2354                                 if (IS_INT_LNG_TYPE(var->type)) {
2355                                         if (!md->params[s3].inmemory) {
2356                                                 s1 = emit_load(jd, iptr, var, d);
2357                                                 M_INTMOVE(s1, d);
2358                                         }
2359                                         else {
2360                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2361                                                 M_LST(s1, REG_SP, d);
2362                                         }
2363                                 }
2364                                 else {
2365                                         if (!md->params[s3].inmemory) {
2366                                                 s1 = emit_load(jd, iptr, var, d);
2367                                                 M_FLTMOVE(s1, d);
2368                                         }
2369                                         else {
2370                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2371
2372                                                 if (IS_2_WORD_TYPE(var->type))
2373                                                         M_DST(s1, REG_SP, d);
2374                                                 else
2375                                                         M_FST(s1, REG_SP, d);
2376                                         }
2377                                 }
2378                         }
2379
2380                         /* generate method profiling code */
2381
2382                         PROFILE_CYCLE_STOP;
2383
2384                         switch (iptr->opc) {
2385                         case ICMD_BUILTIN:
2386                                 if (bte->stub == NULL) {
2387                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2388                                 }
2389                                 else {
2390                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2391                                 }
2392                                 M_CALL(REG_ITMP1);
2393                                 break;
2394
2395                         case ICMD_INVOKESPECIAL:
2396                                 emit_nullpointer_check(cd, iptr, REG_A0);
2397                                 /* fall through */
2398
2399                         case ICMD_INVOKESTATIC:
2400                                 if (lm == NULL) {
2401                                         disp = dseg_add_unique_address(cd, um);
2402
2403                                         patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
2404                                                                                   um, disp);
2405                                 }
2406                                 else {
2407                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2408                                 }
2409
2410                                 M_ALD(REG_ITMP2, RIP, disp);
2411                                 M_CALL(REG_ITMP2);
2412                                 break;
2413
2414                         case ICMD_INVOKEVIRTUAL:
2415                                 if (lm == NULL) {
2416                                         patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
2417
2418                                         s1 = 0;
2419                                 }
2420                                 else {
2421                                         s1 = OFFSET(vftbl_t, table[0]) +
2422                                                 sizeof(methodptr) * lm->vftblindex;
2423                                 }
2424
2425                                 /* implicit null-pointer check */
2426                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2427                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2428                                 M_CALL(REG_ITMP3);
2429                                 break;
2430
2431                         case ICMD_INVOKEINTERFACE:
2432                                 if (lm == NULL) {
2433                                         patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
2434
2435                                         s1 = 0;
2436                                         s2 = 0;
2437                                 }
2438                                 else {
2439                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2440                                                 sizeof(methodptr) * lm->clazz->index;
2441
2442                                         s2 = sizeof(methodptr) * (lm - lm->clazz->methods);
2443                                 }
2444
2445                                 /* implicit null-pointer check */
2446                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2447                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2448                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2449                                 M_CALL(REG_ITMP3);
2450                                 break;
2451                         }
2452
2453                         /* generate method profiling code */
2454
2455                         PROFILE_CYCLE_START;
2456
2457                         /* store size of call code in replacement point */
2458
2459                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2460                         REPLACEMENT_POINT_FORGC_BUILTIN_RETURN(cd, iptr);
2461
2462                         /* store return value */
2463
2464                         switch (md->returntype.type) {
2465                         case TYPE_INT:
2466                         case TYPE_LNG:
2467                         case TYPE_ADR:
2468                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2469                                 M_INTMOVE(REG_RESULT, s1);
2470                                 emit_store_dst(jd, iptr, s1);
2471                                 break;
2472                         case TYPE_FLT:
2473                         case TYPE_DBL:
2474                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2475                                 M_FLTMOVE(REG_FRESULT, s1);
2476                                 emit_store_dst(jd, iptr, s1);
2477                                 break;
2478                         default:
2479                                 /* TYPE_VOID */
2480                                 break;
2481                         }
2482                         break;
2483
2484
2485                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2486
2487                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2488                                 /* object type cast-check */
2489
2490                                 classinfo *super;
2491                                 s4         superindex;
2492
2493                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2494                                         super      = NULL;
2495                                         superindex = 0;
2496                                 }
2497                                 else {
2498                                         super      = iptr->sx.s23.s3.c.cls;
2499                                         superindex = super->index;
2500                                 }
2501
2502                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2503                                         CODEGEN_CRITICAL_SECTION_NEW;
2504
2505                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2506
2507                                 /* if class is not resolved, check which code to call */
2508
2509                                 if (super == NULL) {
2510                                         M_TEST(s1);
2511                                         emit_label_beq(cd, BRANCH_LABEL_1);
2512
2513                                         patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2514                                                                                   iptr->sx.s23.s3.c.ref, 0);
2515
2516                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2517                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2518                                         emit_label_beq(cd, BRANCH_LABEL_2);
2519                                 }
2520
2521                                 /* interface checkcast code */
2522
2523                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2524                                         if (super != NULL) {
2525                                                 M_TEST(s1);
2526                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2527                                         }
2528
2529                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2530
2531                                         if (super == NULL) {
2532                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
2533                                                                                           iptr->sx.s23.s3.c.ref,
2534                                                                                           0);
2535                                         }
2536
2537                                         M_ILD32(REG_ITMP3,
2538                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2539                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2540                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2541
2542                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2543                                                         OFFSET(vftbl_t, interfacetable[0]) -
2544                                                         superindex * sizeof(methodptr*));
2545                                         M_TEST(REG_ITMP3);
2546                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2547
2548                                         if (super == NULL)
2549                                                 emit_label_br(cd, BRANCH_LABEL_4);
2550                                         else
2551                                                 emit_label(cd, BRANCH_LABEL_3);
2552                                 }
2553
2554                                 /* class checkcast code */
2555
2556                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2557                                         if (super == NULL) {
2558                                                 emit_label(cd, BRANCH_LABEL_2);
2559
2560                                                 cr   = iptr->sx.s23.s3.c.ref;
2561                                                 disp = dseg_add_unique_address(cd, cr);
2562
2563                                                 patcher_add_patch_ref(jd,
2564                                                                                           PATCHER_resolve_classref_to_vftbl,
2565                                                                                           cr, disp);
2566                                         }
2567                                         else {
2568                                                 M_TEST(s1);
2569                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2570
2571                                                 disp = dseg_add_address(cd, super->vftbl);
2572                                         }
2573
2574                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2575                                         M_ALD(REG_ITMP3, RIP, disp);
2576
2577                                         CODEGEN_CRITICAL_SECTION_START;
2578
2579                                         M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2580
2581                                         /*                                      if (s1 != REG_ITMP1) { */
2582                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2583                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2584                                         /*                                                                                              REG_ITMP1); */
2585                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2586                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2587                                         /*                                                                                              REG_ITMP3); */
2588                                         /*  #if defined(ENABLE_THREADS) */
2589                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2590                                         /*  #endif */
2591                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2592
2593                                         /*                                      } else { */
2594
2595                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
2596                                         M_ISUB(REG_ITMP3, REG_ITMP2);
2597                                         M_ALD(REG_ITMP3, RIP, disp);
2598                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2599                                         /*                                      } */
2600
2601                                         CODEGEN_CRITICAL_SECTION_END;
2602
2603                                         M_ICMP(REG_ITMP3, REG_ITMP2);
2604                                         emit_classcast_check(cd, iptr, BRANCH_UGT, REG_ITMP3, s1);
2605
2606                                         if (super != NULL)
2607                                                 emit_label(cd, BRANCH_LABEL_5);
2608                                 }
2609
2610                                 if (super == NULL) {
2611                                         emit_label(cd, BRANCH_LABEL_1);
2612                                         emit_label(cd, BRANCH_LABEL_4);
2613                                 }
2614
2615                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2616                         }
2617                         else {
2618                                 /* array type cast-check */
2619
2620                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2621                                 M_INTMOVE(s1, REG_A0);
2622
2623                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2624                                         cr   = iptr->sx.s23.s3.c.ref;
2625                                         disp = dseg_add_unique_address(cd, cr);
2626
2627                                         patcher_add_patch_ref(jd,
2628                                                                                   PATCHER_resolve_classref_to_classinfo,
2629                                                                                   cr, disp);
2630                                 }
2631                                 else {
2632                                         disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2633                                 }
2634
2635                                 M_ALD(REG_A1, RIP, disp);
2636                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2637                                 M_CALL(REG_ITMP1);
2638
2639                                 /* s1 may have been destroyed over the function call */
2640                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2641                                 M_TEST(REG_RESULT);
2642                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2643
2644                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2645                         }
2646
2647                         M_INTMOVE(s1, d);
2648                         emit_store_dst(jd, iptr, d);
2649                         break;
2650
2651                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2652
2653                         {
2654                         classinfo *super;
2655                         s4         superindex;
2656
2657                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2658                                 super      = NULL;
2659                                 superindex = 0;
2660                         }
2661                         else {
2662                                 super      = iptr->sx.s23.s3.c.cls;
2663                                 superindex = super->index;
2664                         }
2665
2666                         if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2667                                 CODEGEN_CRITICAL_SECTION_NEW;
2668
2669                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2670                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2671
2672                         if (s1 == d) {
2673                                 M_INTMOVE(s1, REG_ITMP1);
2674                                 s1 = REG_ITMP1;
2675                         }
2676
2677                         M_CLR(d);
2678
2679                         /* if class is not resolved, check which code to call */
2680
2681                         if (super == NULL) {
2682                                 M_TEST(s1);
2683                                 emit_label_beq(cd, BRANCH_LABEL_1);
2684
2685                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2686                                                                           iptr->sx.s23.s3.c.ref, 0);
2687
2688                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2689                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2690                                 emit_label_beq(cd, BRANCH_LABEL_2);
2691                         }
2692
2693                         /* interface instanceof code */
2694
2695                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2696                                 if (super != NULL) {
2697                                         M_TEST(s1);
2698                                         emit_label_beq(cd, BRANCH_LABEL_3);
2699                                 }
2700
2701                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2702
2703                                 if (super == NULL) {
2704                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
2705                                                                                   iptr->sx.s23.s3.c.ref, 0);
2706                                 }
2707
2708                                 M_ILD32(REG_ITMP3,
2709                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2710                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2711
2712                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2713
2714                                 M_BLE(a);
2715                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2716                                                 OFFSET(vftbl_t, interfacetable[0]) -
2717                                                 superindex * sizeof(methodptr*));
2718                                 M_TEST(REG_ITMP1);
2719                                 M_SETNE(d);
2720
2721                                 if (super == NULL)
2722                                         emit_label_br(cd, BRANCH_LABEL_4);
2723                                 else
2724                                         emit_label(cd, BRANCH_LABEL_3);
2725                         }
2726
2727                         /* class instanceof code */
2728
2729                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2730                                 if (super == NULL) {
2731                                         emit_label(cd, BRANCH_LABEL_2);
2732
2733                                         cr   = iptr->sx.s23.s3.c.ref;
2734                                         disp = dseg_add_unique_address(cd, cr);
2735
2736                                         patcher_add_patch_ref(jd,
2737                                                                                   PATCHER_resolve_classref_to_vftbl,
2738                                                                                   cr, disp);
2739                                 }
2740                                 else {
2741                                         M_TEST(s1);
2742                                         emit_label_beq(cd, BRANCH_LABEL_5);
2743
2744                                         disp = dseg_add_address(cd, super->vftbl);
2745                                 }
2746
2747                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2748                                 M_ALD(REG_ITMP2, RIP, disp);
2749
2750                                 CODEGEN_CRITICAL_SECTION_START;
2751
2752                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
2753                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2754                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2755
2756                                 CODEGEN_CRITICAL_SECTION_END;
2757
2758                                 M_ISUB(REG_ITMP2, REG_ITMP1);
2759                                 M_CLR(d); /* may be REG_ITMP2 */
2760                                 M_ICMP(REG_ITMP3, REG_ITMP1);
2761                                 M_SETULE(d);
2762
2763                                 if (super != NULL)
2764                                         emit_label(cd, BRANCH_LABEL_5);
2765                         }
2766
2767                         if (super == NULL) {
2768                                 emit_label(cd, BRANCH_LABEL_1);
2769                                 emit_label(cd, BRANCH_LABEL_4);
2770                         }
2771
2772                         emit_store_dst(jd, iptr, d);
2773                         }
2774                         break;
2775
2776                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2777
2778                         /* check for negative sizes and copy sizes to stack if necessary  */
2779
2780                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2781
2782                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2783
2784                                 /* copy SAVEDVAR sizes to stack */
2785                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2786
2787                                 /* Already Preallocated? */
2788                                 if (!(var->flags & PREALLOC)) {
2789                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2790                                         M_LST(s2, REG_SP, s1 * 8);
2791                                 }
2792                         }
2793
2794                         /* a0 = dimension count */
2795
2796                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2797
2798                         /* is a patcher function set? */
2799
2800                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2801                                 cr   = iptr->sx.s23.s3.c.ref;
2802                                 disp = dseg_add_unique_address(cd, cr);
2803
2804                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
2805                                                                           cr, disp);
2806                         }
2807                         else {
2808                                 disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2809                         }
2810
2811                         /* a1 = classinfo */
2812
2813                         M_ALD(REG_A1, RIP, disp);
2814
2815                         /* a2 = pointer to dimensions = stack pointer */
2816
2817                         M_MOV(REG_SP, REG_A2);
2818
2819                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2820                         M_CALL(REG_ITMP1);
2821
2822                         /* check for exception before result assignment */
2823
2824                         emit_exception_check(cd, iptr);
2825
2826                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2827                         M_INTMOVE(REG_RESULT, s1);
2828                         emit_store_dst(jd, iptr, s1);
2829                         break;
2830
2831                 default:
2832                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2833                                                                                    iptr->opc);
2834                         return false;
2835         } /* switch */
2836
2837         } /* for instruction */
2838
2839         MCODECHECK(512); /* XXX require a lower number? */
2840
2841         /* At the end of a basic block we may have to append some nops,
2842            because the patcher stub calling code might be longer than the
2843            actual instruction. So codepatching does not change the
2844            following block unintentionally. */
2845
2846         if (cd->mcodeptr < cd->lastmcodeptr) {
2847                 while (cd->mcodeptr < cd->lastmcodeptr) {
2848                         M_NOP;
2849                 }
2850         }
2851
2852         } /* if (bptr -> flags >= BBREACHED) */
2853         } /* for basic block */
2854
2855         /* Generate patcher traps. */
2856
2857         emit_patcher_traps(jd);
2858
2859         /* everything's ok */
2860
2861         return true;
2862 }
2863
2864
2865 /* codegen_emit_stub_native ****************************************************
2866
2867    Emits a stub routine which calls a native method.
2868
2869 *******************************************************************************/
2870
2871 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f, int skipparams)
2872 {
2873         methodinfo  *m;
2874         codeinfo    *code;
2875         codegendata *cd;
2876         methoddesc  *md;
2877         int          i, j;
2878         int          s1, s2;
2879         int          disp;
2880
2881         /* Sanity check. */
2882
2883         assert(f != NULL);
2884
2885         /* Get required compiler data. */
2886
2887         m    = jd->m;
2888         code = jd->code;
2889         cd   = jd->cd;
2890
2891         /* initialize variables */
2892
2893         md = m->parseddesc;
2894
2895         /* calculate stack frame size */
2896
2897         cd->stackframesize =
2898                 sizeof(stackframeinfo_t) / SIZEOF_VOID_P +
2899                 sizeof(localref_table) / SIZEOF_VOID_P +
2900                 md->paramcount +
2901                 (md->returntype.type == TYPE_VOID ? 0 : 1) +
2902                 nmd->memuse;
2903
2904         ALIGN_ODD(cd->stackframesize);              /* keep stack 16-byte aligned */
2905
2906         /* create method header */
2907
2908         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2909         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2910         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2911         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2912         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2913
2914 #if defined(ENABLE_PROFILING)
2915         /* generate native method profiling code */
2916
2917         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
2918                 /* count frequency */
2919
2920                 M_MOV_IMM(code, REG_ITMP3);
2921                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
2922         }
2923 #endif
2924
2925         /* generate stub code */
2926
2927         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2928
2929 #if defined(ENABLE_GC_CACAO)
2930         /* Save callee saved integer registers in stackframeinfo (GC may
2931            need to recover them during a collection). */
2932
2933         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
2934                 OFFSET(stackframeinfo_t, intregs);
2935
2936         for (i = 0; i < INT_SAV_CNT; i++)
2937                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2938 #endif
2939
2940         /* save integer and float argument registers */
2941
2942         for (i = 0; i < md->paramcount; i++) {
2943                 if (!md->params[i].inmemory) {
2944                         s1 = md->params[i].regoff;
2945
2946                         switch (md->paramtypes[i].type) {
2947                         case TYPE_INT:
2948                         case TYPE_LNG:
2949                         case TYPE_ADR:
2950                                 M_LST(s1, REG_SP, i * 8);
2951                                 break;
2952                         case TYPE_FLT:
2953                         case TYPE_DBL:
2954                                 M_DST(s1, REG_SP, i * 8);
2955                                 break;
2956                         }
2957                 }
2958         }
2959
2960         /* create dynamic stack info */
2961
2962         M_MOV(REG_SP, REG_A0);
2963         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
2964         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
2965         M_CALL(REG_ITMP1);
2966
2967         /* remember class argument */
2968
2969         if (m->flags & ACC_STATIC)
2970                 M_MOV(REG_RESULT, REG_ITMP2);
2971
2972         /* restore integer and float argument registers */
2973
2974         for (i = 0; i < md->paramcount; i++) {
2975                 if (!md->params[i].inmemory) {
2976                         s1 = md->params[i].regoff;
2977
2978                         switch (md->paramtypes[i].type) {
2979                         case TYPE_INT:
2980                         case TYPE_LNG:
2981                         case TYPE_ADR:
2982                                 M_LLD(s1, REG_SP, i * 8);
2983                                 break;
2984                         case TYPE_FLT:
2985                         case TYPE_DBL:
2986                                 M_DLD(s1, REG_SP, i * 8);
2987                                 break;
2988                         }
2989                 }
2990         }
2991
2992         /* Copy or spill arguments to new locations. */
2993
2994         for (i = md->paramcount - 1, j = i + skipparams; i >= 0; i--, j--) {
2995                 s2 = nmd->params[j].regoff;
2996
2997                 switch (md->paramtypes[i].type) {
2998                 case TYPE_INT:
2999                 case TYPE_LNG:
3000                 case TYPE_ADR:
3001                         if (!md->params[i].inmemory) {
3002                                 s1 = md->params[i].regoff;
3003
3004                                 if (!nmd->params[j].inmemory)
3005                                         M_INTMOVE(s1, s2);
3006                                 else
3007                                         M_LST(s1, REG_SP, s2);
3008                         }
3009                         else {
3010                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3011                                 M_LLD(REG_ITMP1, REG_SP, s1);
3012                                 M_LST(REG_ITMP1, REG_SP, s2);
3013                         }
3014                         break;
3015                 case TYPE_FLT:
3016                         /* We only copy spilled float arguments, as the float
3017                            argument registers keep unchanged. */
3018
3019                         if (md->params[i].inmemory) {
3020                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3021
3022                                 M_FLD(REG_FTMP1, REG_SP, s1);
3023                                 M_FST(REG_FTMP1, REG_SP, s2);
3024                         }
3025                         break;
3026                 case TYPE_DBL:
3027                         if (md->params[i].inmemory) {
3028                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3029                                 M_DLD(REG_FTMP1, REG_SP, s1);
3030                                 M_DST(REG_FTMP1, REG_SP, s2);
3031                         }
3032                         break;
3033                 }
3034         }
3035
3036         /* Handle native Java methods. */
3037
3038         if (m->flags & ACC_NATIVE) {
3039                 /* put class into second argument register */
3040
3041                 if (m->flags & ACC_STATIC)
3042                         M_MOV(REG_ITMP2, REG_A1);
3043
3044                 /* put env into first argument register */
3045
3046                 M_MOV_IMM(_Jv_env, REG_A0);
3047         }
3048
3049         /* Call the native function. */
3050
3051         disp = dseg_add_functionptr(cd, f);
3052         M_ALD(REG_ITMP1, RIP, disp);
3053         M_CALL(REG_ITMP1);
3054
3055         /* save return value */
3056
3057         switch (md->returntype.type) {
3058         case TYPE_INT:
3059         case TYPE_LNG:
3060         case TYPE_ADR:
3061                 switch (md->returntype.decltype) {
3062                 case PRIMITIVETYPE_BOOLEAN:
3063                         M_BZEXT(REG_RESULT, REG_RESULT);
3064                         break;
3065                 case PRIMITIVETYPE_BYTE:
3066                         M_BSEXT(REG_RESULT, REG_RESULT);
3067                         break;
3068                 case PRIMITIVETYPE_CHAR:
3069                         M_CZEXT(REG_RESULT, REG_RESULT);
3070                         break;
3071                 case PRIMITIVETYPE_SHORT:
3072                         M_SSEXT(REG_RESULT, REG_RESULT);
3073                         break;
3074                 }
3075                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3076                 break;
3077         case TYPE_FLT:
3078         case TYPE_DBL:
3079                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3080                 break;
3081         case TYPE_VOID:
3082                 break;
3083         }
3084
3085         /* remove native stackframe info */
3086
3087         M_MOV(REG_SP, REG_A0);
3088         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3089         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3090         M_CALL(REG_ITMP1);
3091         M_MOV(REG_RESULT, REG_ITMP3);
3092
3093         /* restore return value */
3094
3095         switch (md->returntype.type) {
3096         case TYPE_INT:
3097         case TYPE_LNG:
3098         case TYPE_ADR:
3099                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3100                 break;
3101         case TYPE_FLT:
3102         case TYPE_DBL:
3103                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3104                 break;
3105         case TYPE_VOID:
3106                 break;
3107         }
3108
3109 #if defined(ENABLE_GC_CACAO)
3110         /* Restore callee saved integer registers from stackframeinfo (GC
3111            might have modified them during a collection). */
3112
3113         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
3114                 OFFSET(stackframeinfo_t, intregs);
3115
3116         for (i = 0; i < INT_SAV_CNT; i++)
3117                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3118 #endif
3119
3120         /* remove stackframe */
3121
3122         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3123
3124         /* test for exception */
3125
3126         M_TEST(REG_ITMP3);
3127         M_BNE(1);
3128         M_RET;
3129
3130         /* handle exception */
3131
3132         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3133         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3134         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3135
3136         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3137         M_JMP(REG_ITMP3);
3138 }
3139
3140
3141 /*
3142  * These are local overrides for various environment variables in Emacs.
3143  * Please do not remove this and leave it at the end of the file, where
3144  * Emacs will automagically detect them.
3145  * ---------------------------------------------------------------------
3146  * Local variables:
3147  * mode: c
3148  * indent-tabs-mode: t
3149  * c-basic-offset: 4
3150  * tab-width: 4
3151  * End:
3152  * vim:noexpandtab:sw=4:ts=4:
3153  */