src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007, 2008
   4    CACAOVM - Verein zur Foerderung der freien virtuellen Maschine CACAO
   5
   6    This file is part of CACAO.
   7
   8    This program is free software; you can redistribute it and/or
   9    modify it under the terms of the GNU General Public License as
  10    published by the Free Software Foundation; either version 2, or (at
  11    your option) any later version.
  12
  13    This program is distributed in the hope that it will be useful, but
  14    WITHOUT ANY WARRANTY; without even the implied warranty of
  15    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16    General Public License for more details.
  17
  18    You should have received a copy of the GNU General Public License
  19    along with this program; if not, write to the Free Software
  20    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  21    02110-1301, USA.
  22
  23 */
  24
  25
  26 #include "config.h"
  27
  28 #include <assert.h>
  29 #include <stdio.h>
  30 #include <stdint.h>
  31
  32 #include "vm/types.h"
  33
  34 #include "md-abi.h"
  35
  36 #include "vm/jit/x86_64/arch.h"
  37 #include "vm/jit/x86_64/codegen.h"
  38 #include "vm/jit/x86_64/emit.h"
  39
  40 #include "mm/memory.h"
  41
  42 #include "native/localref.h"
  43 #include "native/native.h"
  44
  45 #include "threads/lock-common.h"
  46
  47 #include "vm/builtin.h"
  48 #include "vm/exceptions.hpp"
  49 #include "vm/global.h"
  50 #include "vm/loader.h"
  51 #include "vm/options.h"
  52 #include "vm/primitive.hpp"
  53 #include "vm/statistics.h"
  54 #include "vm/string.hpp"
  55 #include "vm/vm.hpp"
  56
  57 #include "vm/jit/abi.h"
  58 #include "vm/jit/asmpart.h"
  59 #include "vm/jit/code.h"
  60 #include "vm/jit/codegen-common.h"
  61 #include "vm/jit/dseg.h"
  62 #include "vm/jit/emit-common.h"
  63 #include "vm/jit/jit.h"
  64 #include "vm/jit/linenumbertable.h"
  65 #include "vm/jit/methodheader.h"
  66 #include "vm/jit/parse.h"
  67 #include "vm/jit/patcher-common.h"
  68 #include "vm/jit/reg.h"
  69 #include "vm/jit/replace.h"
  70 #include "vm/jit/stacktrace.hpp"
  71 #include "vm/jit/trap.h"
  72
  73 #if defined(ENABLE_LSRA)
  74 # include "vm/jit/allocator/lsra.h"
  75 #endif
  76
  77
  78 /* codegen_emit ****************************************************************
  79
  80    Generates machine code.
  81
  82 *******************************************************************************/
  83
  84 bool codegen_emit(jitdata *jd)
  85 {
  86         methodinfo         *m;
  87         codeinfo           *code;
  88         codegendata        *cd;
  89         registerdata       *rd;
  90         s4                  len, s1, s2, s3, d, disp;
  91         u2                  currentline;
  92         ptrint              a;
  93         varinfo            *var, *dst;
  94         basicblock         *bptr;
  95         instruction        *iptr;
  96         constant_classref  *cr;
  97         unresolved_class   *uc;
  98         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
  99         unresolved_method  *um;
 100         builtintable_entry *bte;
 101         methoddesc         *md;
 102         fieldinfo          *fi;
 103         unresolved_field   *uf;
 104         s4                  fieldtype;
 105         s4                 varindex;
 106
 107         /* get required compiler data */
 108
 109         m    = jd->m;
 110         code = jd->code;
 111         cd   = jd->cd;
 112         rd   = jd->rd;
 113
 114         /* prevent compiler warnings */
 115
 116         d   = 0;
 117         lm  = NULL;
 118         um  = NULL;
 119         bte = NULL;
 120
 121         {
 122         s4 i, p, t, l;
 123         s4 savedregs_num;
 124
 125         savedregs_num = 0;
 126
 127         /* space to save used callee saved registers */
 128
 129         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 130         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 131
 132         cd->stackframesize = rd->memuse + savedregs_num;
 133
 134 #if defined(ENABLE_THREADS)
 135         /* space to save argument of monitor_enter */
 136
 137         if (checksync && code_is_synchronized(code))
 138                 cd->stackframesize++;
 139 #endif
 140
 141         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 142            native code e.g. libc or jni (alignment problems with
 143            movaps). */
 144
 145         if (!code_is_leafmethod(code) || opt_verbosecall)
 146                 cd->stackframesize |= 0x1;
 147
 148         /* create method header */
 149
 150         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 151         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 152
 153         code->synchronizedoffset = rd->memuse * 8;
 154
 155         if (code_is_leafmethod(code))
 156                 (void) dseg_add_unique_s4(cd, 1);                  /* IsLeaf          */
 157         else
 158                 (void) dseg_add_unique_s4(cd, 0);                  /* IsLeaf          */
 159
 160         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 161         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 162
 163 #if defined(ENABLE_PROFILING)
 164         /* generate method profiling code */
 165
 166         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 167                 /* count frequency */
 168
 169                 M_MOV_IMM(code, REG_ITMP3);
 170                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 171
 172                 PROFILE_CYCLE_START;
 173         }
 174 #endif
 175
 176         /* create stack frame (if necessary) */
 177
 178         if (cd->stackframesize)
 179                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 180
 181         /* save used callee saved registers */
 182
 183         p = cd->stackframesize;
 184         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 185                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 186         }
 187         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 188                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 189         }
 190
 191         /* take arguments out of register or stack frame */
 192
 193         md = m->parseddesc;
 194
 195         for (p = 0, l = 0; p < md->paramcount; p++) {
 196                 t = md->paramtypes[p].type;
 197
 198                 varindex = jd->local_map[l * 5 + t];
 199
 200                 l++;
 201                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 202                         l++;
 203
 204                 if (varindex == UNUSED)
 205                         continue;
 206
 207                 var = VAR(varindex);
 208
 209                 s1 = md->params[p].regoff;
 210
 211                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 212                         if (!md->params[p].inmemory) {           /* register arguments    */
 213                                 if (!IS_INMEMORY(var->flags))
 214                                         M_INTMOVE(s1, var->vv.regoff);
 215                                 else
 216                                     M_LST(s1, REG_SP, var->vv.regoff);
 217                         }
 218                         else {                                 /* stack arguments       */
 219                                 if (!IS_INMEMORY(var->flags))
 220                                         /* + 8 for return address */
 221                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 222                                 else
 223                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 224                         }
 225                 }
 226                 else {                                     /* floating args         */
 227                         if (!md->params[p].inmemory) {           /* register arguments    */
 228                                 if (!IS_INMEMORY(var->flags))
 229                                         M_FLTMOVE(s1, var->vv.regoff);
 230                                 else
 231                                         M_DST(s1, REG_SP, var->vv.regoff);
 232                         }
 233                         else {                                 /* stack arguments       */
 234                                 if (!IS_INMEMORY(var->flags))
 235                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 236                                 else
 237                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 238                         }
 239                 }
 240         }
 241
 242         /* save monitorenter argument */
 243
 244 #if defined(ENABLE_THREADS)
 245         if (checksync && code_is_synchronized(code)) {
 246                 /* stack offset for monitor argument */
 247
 248                 s1 = rd->memuse;
 249
 250                 if (opt_verbosecall) {
 251                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 252
 253                         for (p = 0; p < INT_ARG_CNT; p++)
 254                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 255
 256                         for (p = 0; p < FLT_ARG_CNT; p++)
 257                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 258
 259                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 260                 }
 261
 262                 /* decide which monitor enter function to call */
 263
 264                 if (m->flags & ACC_STATIC) {
 265                         M_MOV_IMM(&m->clazz->object.header, REG_A0);
 266                 }
 267                 else {
 268                         M_TEST(REG_A0);
 269                         M_BNE(8);
 270                         M_ALD_MEM(REG_A0, TRAP_NullPointerException);
 271                 }
 272
 273                 M_AST(REG_A0, REG_SP, s1 * 8);
 274                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 275                 M_CALL(REG_ITMP1);
 276
 277                 if (opt_verbosecall) {
 278                         for (p = 0; p < INT_ARG_CNT; p++)
 279                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 280
 281                         for (p = 0; p < FLT_ARG_CNT; p++)
 282                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 283
 284                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 285                 }
 286         }
 287 #endif
 288
 289 #if !defined(NDEBUG)
 290         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 291                 emit_verbosecall_enter(jd);
 292 #endif /* !defined(NDEBUG) */
 293
 294         }
 295
 296         /* end of header generation */
 297
 298         /* create replacement points */
 299
 300         REPLACEMENT_POINTS_INIT(cd, jd);
 301
 302         /* walk through all basic blocks */
 303
 304         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 305
 306                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 307
 308                 if (bptr->flags >= BBREACHED) {
 309
 310                 /* branch resolving */
 311
 312                 codegen_resolve_branchrefs(cd, bptr);
 313
 314                 /* handle replacement points */
 315
 316                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 317
 318                 /* copy interface registers to their destination */
 319
 320                 len = bptr->indepth;
 321                 MCODECHECK(512);
 322
 323 #if defined(ENABLE_PROFILING)
 324                 /* generate basicblock profiling code */
 325
 326                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 327                         /* count frequency */
 328
 329                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 330                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 331
 332                         /* if this is an exception handler, start profiling again */
 333
 334                         if (bptr->type == BBTYPE_EXH)
 335                                 PROFILE_CYCLE_START;
 336                 }
 337 #endif
 338
 339 #if defined(ENABLE_LSRA)
 340                 if (opt_lsra) {
 341                         while (len) {
 342                                 len--;
 343                                 src = bptr->invars[len];
 344                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 345                                         if (bptr->type == BBTYPE_EXH) {
 346 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 347                                                 if (!IS_INMEMORY(src->flags))
 348                                                         d= src->vv.regoff;
 349                                                 else
 350                                                         d=REG_ITMP1;
 351                                                 M_INTMOVE(REG_ITMP1, d);
 352                                                 emit_store(jd, NULL, src, d);
 353                                         }
 354                                 }
 355                         }
 356
 357                 } else {
 358 #endif
 359
 360                 while (len) {
 361                         len--;
 362                         var = VAR(bptr->invars[len]);
 363                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 364                                 if (bptr->type == BBTYPE_EXH) {
 365                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 366                                         M_INTMOVE(REG_ITMP1, d);
 367                                         emit_store(jd, NULL, var, d);
 368                                 }
 369                         }
 370                         else {
 371                                 assert((var->flags & INOUT));
 372                         }
 373                 }
 374 #if defined(ENABLE_LSRA)
 375                 }
 376 #endif
 377                 /* walk through all instructions */
 378
 379                 len = bptr->icount;
 380                 currentline = 0;
 381
 382                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 383                         if (iptr->line != currentline) {
 384                                 linenumbertable_list_entry_add(cd, iptr->line);
 385                                 currentline = iptr->line;
 386                         }
 387
 388                         MCODECHECK(1024);                         /* 1KB should be enough */
 389
 390                 switch (iptr->opc) {
 391                 case ICMD_NOP:        /* ...  ==> ...                                 */
 392                 case ICMD_POP:        /* ..., value  ==> ...                          */
 393                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 394                         break;
 395
 396                 case ICMD_INLINE_START:
 397
 398                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 399                         break;
 400
 401                 case ICMD_INLINE_BODY:
 402
 403                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 404                         linenumbertable_list_entry_add_inline_start(cd, iptr);
 405                         linenumbertable_list_entry_add(cd, iptr->line);
 406                         break;
 407
 408                 case ICMD_INLINE_END:
 409
 410                         linenumbertable_list_entry_add_inline_end(cd, iptr);
 411                         linenumbertable_list_entry_add(cd, iptr->line);
 412                         break;
 413
 414                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 415
 416                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 417                         emit_nullpointer_check(cd, iptr, s1);
 418                         break;
 419
 420                 /* constant operations ************************************************/
 421
 422                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 423
 424                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 425                         ICONST(d, iptr->sx.val.i);
 426                         emit_store_dst(jd, iptr, d);
 427                         break;
 428
 429                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 430
 431                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 432                         LCONST(d, iptr->sx.val.l);
 433                         emit_store_dst(jd, iptr, d);
 434                         break;
 435
 436                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 437
 438                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 439                         disp = dseg_add_float(cd, iptr->sx.val.f);
 440                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 441                         emit_store_dst(jd, iptr, d);
 442                         break;
 443
 444                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 445
 446                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 447                         disp = dseg_add_double(cd, iptr->sx.val.d);
 448                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 449                         emit_store_dst(jd, iptr, d);
 450                         break;
 451
 452                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 453
 454                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 455
 456                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 457                                 cr   = iptr->sx.val.c.ref;
 458                                 disp = dseg_add_unique_address(cd, cr);
 459
 460 /*                              PROFILE_CYCLE_STOP; */
 461
 462                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
 463                                                                           cr, disp);
 464
 465 /*                              PROFILE_CYCLE_START; */
 466
 467                                 M_ALD(d, RIP, disp);
 468                         }
 469                         else {
 470                                 if (iptr->sx.val.anyptr == 0) {
 471                                         M_CLR(d);
 472                                 }
 473                                 else {
 474                                         disp = dseg_add_address(cd, iptr->sx.val.anyptr);
 475                                         M_ALD(d, RIP, disp);
 476                                 }
 477                         }
 478                         emit_store_dst(jd, iptr, d);
 479                         break;
 480
 481
 482                 /* load/store/copy/move operations ************************************/
 483
 484                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 485                 case ICMD_ALOAD:      /* s1 = local variable                          */
 486                 case ICMD_LLOAD:
 487                 case ICMD_FLOAD:
 488                 case ICMD_DLOAD:
 489                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 490                 case ICMD_LSTORE:
 491                 case ICMD_FSTORE:
 492                 case ICMD_DSTORE:
 493                 case ICMD_COPY:
 494                 case ICMD_MOVE:
 495
 496                         emit_copy(jd, iptr);
 497                         break;
 498
 499                 case ICMD_ASTORE:
 500                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 501                                 emit_copy(jd, iptr);
 502                         break;
 503
 504                 /* integer operations *************************************************/
 505
 506                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 507
 508                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 509                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 510                         M_INTMOVE(s1, d);
 511                         M_INEG(d);
 512                         emit_store_dst(jd, iptr, d);
 513                         break;
 514
 515                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 516
 517                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 518                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 519                         M_INTMOVE(s1, d);
 520                         M_LNEG(d);
 521                         emit_store_dst(jd, iptr, d);
 522                         break;
 523
 524                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 525
 526                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 527                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 528                         M_ISEXT(s1, d);
 529                         emit_store_dst(jd, iptr, d);
 530                         break;
 531
 532                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 533
 534                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 535                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 536                         M_IMOV(s1, d);
 537                         emit_store_dst(jd, iptr, d);
 538                         break;
 539
 540                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 541
 542                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 543                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 544                         M_BSEXT(s1, d);
 545                         emit_store_dst(jd, iptr, d);
 546                         break;
 547
 548                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 549
 550                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 551                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 552                         M_CZEXT(s1, d);
 553                         emit_store_dst(jd, iptr, d);
 554                         break;
 555
 556                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 557
 558                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 559                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 560                         M_SSEXT(s1, d);
 561                         emit_store_dst(jd, iptr, d);
 562                         break;
 563
 564
 565                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 566
 567                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 568                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 569                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 570                         if (s2 == d)
 571                                 M_IADD(s1, d);
 572                         else {
 573                                 M_INTMOVE(s1, d);
 574                                 M_IADD(s2, d);
 575                         }
 576                         emit_store_dst(jd, iptr, d);
 577                         break;
 578
 579                 case ICMD_IINC:
 580                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 581                                       /* sx.val.i = constant                             */
 582
 583                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 584                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 585
 586                         /* Using inc and dec is not faster than add (tested with
 587                            sieve). */
 588
 589                         M_INTMOVE(s1, d);
 590                         M_IADD_IMM(iptr->sx.val.i, d);
 591                         emit_store_dst(jd, iptr, d);
 592                         break;
 593
 594                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 595
 596                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 597                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 598                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 599                         if (s2 == d)
 600                                 M_LADD(s1, d);
 601                         else {
 602                                 M_INTMOVE(s1, d);
 603                                 M_LADD(s2, d);
 604                         }
 605                         emit_store_dst(jd, iptr, d);
 606                         break;
 607
 608                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 609                                       /* sx.val.l = constant                             */
 610
 611                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 612                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 613                         M_INTMOVE(s1, d);
 614                         if (IS_IMM32(iptr->sx.val.l))
 615                                 M_LADD_IMM(iptr->sx.val.l, d);
 616                         else {
 617                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 618                                 M_LADD(REG_ITMP2, d);
 619                         }
 620                         emit_store_dst(jd, iptr, d);
 621                         break;
 622
 623                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 624
 625                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 626                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 627                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 628                         if (s2 == d) {
 629                                 M_INTMOVE(s1, REG_ITMP1);
 630                                 M_ISUB(s2, REG_ITMP1);
 631                                 M_INTMOVE(REG_ITMP1, d);
 632                         } else {
 633                                 M_INTMOVE(s1, d);
 634                                 M_ISUB(s2, d);
 635                         }
 636                         emit_store_dst(jd, iptr, d);
 637                         break;
 638
 639                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 640                                       /* sx.val.i = constant                             */
 641
 642                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 643                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 644                         M_INTMOVE(s1, d);
 645                         M_ISUB_IMM(iptr->sx.val.i, d);
 646                         emit_store_dst(jd, iptr, d);
 647                         break;
 648
 649                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 650
 651                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 652                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 653                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 654                         if (s2 == d) {
 655                                 M_INTMOVE(s1, REG_ITMP1);
 656                                 M_LSUB(s2, REG_ITMP1);
 657                                 M_INTMOVE(REG_ITMP1, d);
 658                         } else {
 659                                 M_INTMOVE(s1, d);
 660                                 M_LSUB(s2, d);
 661                         }
 662                         emit_store_dst(jd, iptr, d);
 663                         break;
 664
 665                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 666                                       /* sx.val.l = constant                             */
 667
 668                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 669                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 670                         M_INTMOVE(s1, d);
 671                         if (IS_IMM32(iptr->sx.val.l))
 672                                 M_LSUB_IMM(iptr->sx.val.l, d);
 673                         else {
 674                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 675                                 M_LSUB(REG_ITMP2, d);
 676                         }
 677                         emit_store_dst(jd, iptr, d);
 678                         break;
 679
 680                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 681
 682                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 683                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 684                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 685                         if (s2 == d)
 686                                 M_IMUL(s1, d);
 687                         else {
 688                                 M_INTMOVE(s1, d);
 689                                 M_IMUL(s2, d);
 690                         }
 691                         emit_store_dst(jd, iptr, d);
 692                         break;
 693
 694                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 695                                       /* sx.val.i = constant                             */
 696
 697                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 698                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 699                         if (iptr->sx.val.i == 2) {
 700                                 M_INTMOVE(s1, d);
 701                                 M_ISLL_IMM(1, d);
 702                         } else
 703                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 704                         emit_store_dst(jd, iptr, d);
 705                         break;
 706
 707                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 708
 709                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 710                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 711                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 712                         if (s2 == d)
 713                                 M_LMUL(s1, d);
 714                         else {
 715                                 M_INTMOVE(s1, d);
 716                                 M_LMUL(s2, d);
 717                         }
 718                         emit_store_dst(jd, iptr, d);
 719                         break;
 720
 721                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 722                                       /* sx.val.l = constant                             */
 723
 724                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 725                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 726                         if (IS_IMM32(iptr->sx.val.l))
 727                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 728                         else {
 729                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 730                                 M_INTMOVE(s1, d);
 731                                 M_LMUL(REG_ITMP2, d);
 732                         }
 733                         emit_store_dst(jd, iptr, d);
 734                         break;
 735
 736                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 737
 738                         s1 = emit_load_s1(jd, iptr, RAX);
 739                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 740                         d = codegen_reg_of_dst(jd, iptr, RAX);
 741
 742                         M_INTMOVE(s1, RAX);
 743                         M_INTMOVE(s2, REG_ITMP3);
 744                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 745
 746                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 747
 748                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 749                         M_BNE(4 + 6);
 750                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 751                         M_BEQ(1 + 3);                                          /* 6 bytes */
 752
 753                         emit_cltd(cd);                                         /* 1 byte  */
 754                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 755
 756                         M_INTMOVE(RAX, d);
 757                         emit_store_dst(jd, iptr, d);
 758                         dst = VAROP(iptr->dst);
 759                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 760                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 761                         break;
 762
 763                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 764
 765                         s1 = emit_load_s1(jd, iptr, RAX);
 766                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 767                         d = codegen_reg_of_dst(jd, iptr, RDX);
 768
 769                         M_INTMOVE(s1, RAX);
 770                         M_INTMOVE(s2, REG_ITMP3);
 771                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 772
 773                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 774
 775                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 776                         M_BNE(3 + 4 + 6);
 777                         M_CLR(RDX);                                            /* 3 bytes */
 778                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 779                         M_BEQ(1 + 3);                                          /* 6 bytes */
 780
 781                         emit_cltd(cd);                                         /* 1 byte  */
 782                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 783
 784                         M_INTMOVE(RDX, d);
 785                         emit_store_dst(jd, iptr, d);
 786                         dst = VAROP(iptr->dst);
 787                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 788                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 789                         break;
 790
 791                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 792                                       /* sx.val.i = constant                             */
 793
 794                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 795                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 796                         M_INTMOVE(s1, REG_ITMP1);
 797                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 798                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 799                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 800                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 801                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 802                         emit_store_dst(jd, iptr, d);
 803                         break;
 804
 805                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 806                                       /* sx.val.i = constant                             */
 807
 808                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 809                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 810                         M_INTMOVE(s1, REG_ITMP1);
 811                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 812                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 813                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 814                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 815                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 816                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 817                         emit_store_dst(jd, iptr, d);
 818                         break;
 819
 820
 821                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 822
 823                         s1 = emit_load_s1(jd, iptr, RAX);
 824                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 825                         d = codegen_reg_of_dst(jd, iptr, RAX);
 826
 827                         M_INTMOVE(s1, RAX);
 828                         M_INTMOVE(s2, REG_ITMP3);
 829                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 830
 831                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 832
 833                         /* check as described in jvm spec */
 834                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 835                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 836                         M_BNE(4 + 6);
 837                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 838                         M_BEQ(2 + 3);                                          /* 6 bytes */
 839
 840                         emit_cqto(cd);                                         /* 2 bytes */
 841                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 842
 843                         M_INTMOVE(RAX, d);
 844                         emit_store_dst(jd, iptr, d);
 845                         dst = VAROP(iptr->dst);
 846                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 847                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 848                         break;
 849
 850                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 851
 852                         s1 = emit_load_s1(jd, iptr, RAX);
 853                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 854                         d = codegen_reg_of_dst(jd, iptr, RDX);
 855
 856                         M_INTMOVE(s1, RAX);
 857                         M_INTMOVE(s2, REG_ITMP3);
 858                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 859
 860                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 861
 862                         /* check as described in jvm spec */
 863                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 864                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 865                         M_BNE(3 + 4 + 6);
 866                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 867                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 868                         M_BEQ(2 + 3);                                          /* 6 bytes */
 869
 870                         emit_cqto(cd);                                         /* 2 bytes */
 871                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 872
 873                         M_INTMOVE(RDX, d);
 874                         emit_store_dst(jd, iptr, d);
 875                         dst = VAROP(iptr->dst);
 876                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 877                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 878                         break;
 879
 880                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 881                                       /* sx.val.i = constant                             */
 882
 883                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 884                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 885                         M_INTMOVE(s1, REG_ITMP1);
 886                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 887                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 888                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 889                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 890                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 891                         emit_store_dst(jd, iptr, d);
 892                         break;
 893
 894                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 895                                       /* sx.val.l = constant                             */
 896
 897                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 898                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 899                         M_INTMOVE(s1, REG_ITMP1);
 900                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 901                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 902                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 903                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 904                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 905                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 906                         emit_store_dst(jd, iptr, d);
 907                         break;
 908
 909                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 910
 911                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 912                         emit_ishift(jd, SHIFT_SHL, iptr);
 913                         break;
 914
 915                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 916                                       /* sx.val.i = constant                             */
 917
 918                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 919                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 920                         M_INTMOVE(s1, d);
 921                         M_ISLL_IMM(iptr->sx.val.i, d);
 922                         emit_store_dst(jd, iptr, d);
 923                         break;
 924
 925                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 926
 927                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 928                         emit_ishift(jd, SHIFT_SAR, iptr);
 929                         break;
 930
 931                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 932                                       /* sx.val.i = constant                             */
 933
 934                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 935                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 936                         M_INTMOVE(s1, d);
 937                         M_ISRA_IMM(iptr->sx.val.i, d);
 938                         emit_store_dst(jd, iptr, d);
 939                         break;
 940
 941                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 942
 943                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 944                         emit_ishift(jd, SHIFT_SHR, iptr);
 945                         break;
 946
 947                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 948                                       /* sx.val.i = constant                             */
 949
 950                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 951                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 952                         M_INTMOVE(s1, d);
 953                         M_ISRL_IMM(iptr->sx.val.i, d);
 954                         emit_store_dst(jd, iptr, d);
 955                         break;
 956
 957                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 958
 959                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 960                         emit_lshift(jd, SHIFT_SHL, iptr);
 961                         break;
 962
 963         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 964                                           /* sx.val.i = constant                             */
 965
 966                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 967                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 968                         M_INTMOVE(s1, d);
 969                         M_LSLL_IMM(iptr->sx.val.i, d);
 970                         emit_store_dst(jd, iptr, d);
 971                         break;
 972
 973                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 974
 975                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 976                         emit_lshift(jd, SHIFT_SAR, iptr);
 977                         break;
 978
 979                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 980                                       /* sx.val.i = constant                             */
 981
 982                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 983                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 984                         M_INTMOVE(s1, d);
 985                         M_LSRA_IMM(iptr->sx.val.i, d);
 986                         emit_store_dst(jd, iptr, d);
 987                         break;
 988
 989                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 990
 991                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 992                         emit_lshift(jd, SHIFT_SHR, iptr);
 993                         break;
 994
 995                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 996                                       /* sx.val.l = constant                             */
 997
 998                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 999                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1000                         M_INTMOVE(s1, d);
1001                         M_LSRL_IMM(iptr->sx.val.i, d);
1002                         emit_store_dst(jd, iptr, d);
1003                         break;
1004
1005                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1006
1007                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1008                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1009                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1010                         if (s2 == d)
1011                                 M_IAND(s1, d);
1012                         else {
1013                                 M_INTMOVE(s1, d);
1014                                 M_IAND(s2, d);
1015                         }
1016                         emit_store_dst(jd, iptr, d);
1017                         break;
1018
1019                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1020                                       /* sx.val.i = constant                             */
1021
1022                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1023                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1024                         M_INTMOVE(s1, d);
1025                         M_IAND_IMM(iptr->sx.val.i, d);
1026                         emit_store_dst(jd, iptr, d);
1027                         break;
1028
1029                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1030
1031                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1032                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1033                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1034                         if (s2 == d)
1035                                 M_LAND(s1, d);
1036                         else {
1037                                 M_INTMOVE(s1, d);
1038                                 M_LAND(s2, d);
1039                         }
1040                         emit_store_dst(jd, iptr, d);
1041                         break;
1042
1043                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1044                                       /* sx.val.l = constant                             */
1045
1046                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1047                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1048                         M_INTMOVE(s1, d);
1049                         if (IS_IMM32(iptr->sx.val.l))
1050                                 M_LAND_IMM(iptr->sx.val.l, d);
1051                         else {
1052                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1053                                 M_LAND(REG_ITMP2, d);
1054                         }
1055                         emit_store_dst(jd, iptr, d);
1056                         break;
1057
1058                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1059
1060                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1061                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1062                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1063                         if (s2 == d)
1064                                 M_IOR(s1, d);
1065                         else {
1066                                 M_INTMOVE(s1, d);
1067                                 M_IOR(s2, d);
1068                         }
1069                         emit_store_dst(jd, iptr, d);
1070                         break;
1071
1072                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1073                                       /* sx.val.i = constant                             */
1074
1075                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1076                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1077                         M_INTMOVE(s1, d);
1078                         M_IOR_IMM(iptr->sx.val.i, d);
1079                         emit_store_dst(jd, iptr, d);
1080                         break;
1081
1082                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1083
1084                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1085                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1086                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1087                         if (s2 == d)
1088                                 M_LOR(s1, d);
1089                         else {
1090                                 M_INTMOVE(s1, d);
1091                                 M_LOR(s2, d);
1092                         }
1093                         emit_store_dst(jd, iptr, d);
1094                         break;
1095
1096                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1097                                       /* sx.val.l = constant                             */
1098
1099                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1100                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1101                         M_INTMOVE(s1, d);
1102                         if (IS_IMM32(iptr->sx.val.l))
1103                                 M_LOR_IMM(iptr->sx.val.l, d);
1104                         else {
1105                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1106                                 M_LOR(REG_ITMP2, d);
1107                         }
1108                         emit_store_dst(jd, iptr, d);
1109                         break;
1110
1111                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1112
1113                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1114                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1115                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1116                         if (s2 == d)
1117                                 M_IXOR(s1, d);
1118                         else {
1119                                 M_INTMOVE(s1, d);
1120                                 M_IXOR(s2, d);
1121                         }
1122                         emit_store_dst(jd, iptr, d);
1123                         break;
1124
1125                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1126                                       /* sx.val.i = constant                             */
1127
1128                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1129                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1130                         M_INTMOVE(s1, d);
1131                         M_IXOR_IMM(iptr->sx.val.i, d);
1132                         emit_store_dst(jd, iptr, d);
1133                         break;
1134
1135                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1136
1137                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1138                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1139                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1140                         if (s2 == d)
1141                                 M_LXOR(s1, d);
1142                         else {
1143                                 M_INTMOVE(s1, d);
1144                                 M_LXOR(s2, d);
1145                         }
1146                         emit_store_dst(jd, iptr, d);
1147                         break;
1148
1149                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1150                                       /* sx.val.l = constant                             */
1151
1152                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1153                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1154                         M_INTMOVE(s1, d);
1155                         if (IS_IMM32(iptr->sx.val.l))
1156                                 M_LXOR_IMM(iptr->sx.val.l, d);
1157                         else {
1158                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1159                                 M_LXOR(REG_ITMP2, d);
1160                         }
1161                         emit_store_dst(jd, iptr, d);
1162                         break;
1163
1164
1165                 /* floating operations ************************************************/
1166
1167                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1168
1169                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1170                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1171                         disp = dseg_add_s4(cd, 0x80000000);
1172                         M_FLTMOVE(s1, d);
1173                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1174                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1175                         emit_store_dst(jd, iptr, d);
1176                         break;
1177
1178                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1179
1180                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1181                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1182                         disp = dseg_add_s8(cd, 0x8000000000000000);
1183                         M_FLTMOVE(s1, d);
1184                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1185                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1186                         emit_store_dst(jd, iptr, d);
1187                         break;
1188
1189                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1190
1191                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1192                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1193                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1194                         if (s2 == d)
1195                                 M_FADD(s1, d);
1196                         else {
1197                                 M_FLTMOVE(s1, d);
1198                                 M_FADD(s2, d);
1199                         }
1200                         emit_store_dst(jd, iptr, d);
1201                         break;
1202
1203                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1204
1205                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1206                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1207                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1208                         if (s2 == d)
1209                                 M_DADD(s1, d);
1210                         else {
1211                                 M_FLTMOVE(s1, d);
1212                                 M_DADD(s2, d);
1213                         }
1214                         emit_store_dst(jd, iptr, d);
1215                         break;
1216
1217                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1218
1219                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1220                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1221                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1222                         if (s2 == d) {
1223                                 M_FLTMOVE(s2, REG_FTMP2);
1224                                 s2 = REG_FTMP2;
1225                         }
1226                         M_FLTMOVE(s1, d);
1227                         M_FSUB(s2, d);
1228                         emit_store_dst(jd, iptr, d);
1229                         break;
1230
1231                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1232
1233                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1234                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1235                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1236                         if (s2 == d) {
1237                                 M_FLTMOVE(s2, REG_FTMP2);
1238                                 s2 = REG_FTMP2;
1239                         }
1240                         M_FLTMOVE(s1, d);
1241                         M_DSUB(s2, d);
1242                         emit_store_dst(jd, iptr, d);
1243                         break;
1244
1245                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1246
1247                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1248                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1249                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1250                         if (s2 == d)
1251                                 M_FMUL(s1, d);
1252                         else {
1253                                 M_FLTMOVE(s1, d);
1254                                 M_FMUL(s2, d);
1255                         }
1256                         emit_store_dst(jd, iptr, d);
1257                         break;
1258
1259                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1260
1261                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1262                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1263                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1264                         if (s2 == d)
1265                                 M_DMUL(s1, d);
1266                         else {
1267                                 M_FLTMOVE(s1, d);
1268                                 M_DMUL(s2, d);
1269                         }
1270                         emit_store_dst(jd, iptr, d);
1271                         break;
1272
1273                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1274
1275                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1276                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1277                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1278                         if (s2 == d) {
1279                                 M_FLTMOVE(s2, REG_FTMP2);
1280                                 s2 = REG_FTMP2;
1281                         }
1282                         M_FLTMOVE(s1, d);
1283                         M_FDIV(s2, d);
1284                         emit_store_dst(jd, iptr, d);
1285                         break;
1286
1287                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1288
1289                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1290                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1291                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1292                         if (s2 == d) {
1293                                 M_FLTMOVE(s2, REG_FTMP2);
1294                                 s2 = REG_FTMP2;
1295                         }
1296                         M_FLTMOVE(s1, d);
1297                         M_DDIV(s2, d);
1298                         emit_store_dst(jd, iptr, d);
1299                         break;
1300
1301                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1302
1303                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1304                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1305                         M_CVTIF(s1, d);
1306                         emit_store_dst(jd, iptr, d);
1307                         break;
1308
1309                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1310
1311                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1312                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1313                         M_CVTID(s1, d);
1314                         emit_store_dst(jd, iptr, d);
1315                         break;
1316
1317                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1318
1319                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1320                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1321                         M_CVTLF(s1, d);
1322                         emit_store_dst(jd, iptr, d);
1323                         break;
1324
1325                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1326
1327                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1328                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1329                         M_CVTLD(s1, d);
1330                         emit_store_dst(jd, iptr, d);
1331                         break;
1332
1333                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1334
1335                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1336                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1337                         M_CVTFI(s1, d);
1338                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1339                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1340                                 ((REG_RESULT == d) ? 0 : 3);
1341                         M_BNE(disp);
1342                         M_FLTMOVE(s1, REG_FTMP1);
1343                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1344                         M_CALL(REG_ITMP2);
1345                         M_INTMOVE(REG_RESULT, d);
1346                         emit_store_dst(jd, iptr, d);
1347                         break;
1348
1349                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1350
1351                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1352                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1353                         M_CVTDI(s1, d);
1354                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1355                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1356                                 ((REG_RESULT == d) ? 0 : 3);
1357                         M_BNE(disp);
1358                         M_FLTMOVE(s1, REG_FTMP1);
1359                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1360                         M_CALL(REG_ITMP2);
1361                         M_INTMOVE(REG_RESULT, d);
1362                         emit_store_dst(jd, iptr, d);
1363                         break;
1364
1365                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1366
1367                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1368                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1369                         M_CVTFL(s1, d);
1370                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1371                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1372                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1373                                 ((REG_RESULT == d) ? 0 : 3);
1374                         M_BNE(disp);
1375                         M_FLTMOVE(s1, REG_FTMP1);
1376                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1377                         M_CALL(REG_ITMP2);
1378                         M_INTMOVE(REG_RESULT, d);
1379                         emit_store_dst(jd, iptr, d);
1380                         break;
1381
1382                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1383
1384                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1385                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1386                         M_CVTDL(s1, d);
1387                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1388                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1389                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1390                                 ((REG_RESULT == d) ? 0 : 3);
1391                         M_BNE(disp);
1392                         M_FLTMOVE(s1, REG_FTMP1);
1393                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1394                         M_CALL(REG_ITMP2);
1395                         M_INTMOVE(REG_RESULT, d);
1396                         emit_store_dst(jd, iptr, d);
1397                         break;
1398
1399                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1400
1401                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1402                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1403                         M_CVTFD(s1, d);
1404                         emit_store_dst(jd, iptr, d);
1405                         break;
1406
1407                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1408
1409                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1410                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1411                         M_CVTDF(s1, d);
1412                         emit_store_dst(jd, iptr, d);
1413                         break;
1414
1415                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1416                                           /* == => 0, < => 1, > => -1 */
1417
1418                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1419                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1420                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1421                         M_CLR(d);
1422                         M_MOV_IMM(1, REG_ITMP1);
1423                         M_MOV_IMM(-1, REG_ITMP2);
1424                         emit_ucomiss_reg_reg(cd, s1, s2);
1425                         M_CMOVULT(REG_ITMP1, d);
1426                         M_CMOVUGT(REG_ITMP2, d);
1427                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1428                         emit_store_dst(jd, iptr, d);
1429                         break;
1430
1431                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1432                                           /* == => 0, < => 1, > => -1 */
1433
1434                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1435                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1436                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1437                         M_CLR(d);
1438                         M_MOV_IMM(1, REG_ITMP1);
1439                         M_MOV_IMM(-1, REG_ITMP2);
1440                         emit_ucomiss_reg_reg(cd, s1, s2);
1441                         M_CMOVULT(REG_ITMP1, d);
1442                         M_CMOVUGT(REG_ITMP2, d);
1443                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1444                         emit_store_dst(jd, iptr, d);
1445                         break;
1446
1447                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1448                                           /* == => 0, < => 1, > => -1 */
1449
1450                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1451                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1452                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1453                         M_CLR(d);
1454                         M_MOV_IMM(1, REG_ITMP1);
1455                         M_MOV_IMM(-1, REG_ITMP2);
1456                         emit_ucomisd_reg_reg(cd, s1, s2);
1457                         M_CMOVULT(REG_ITMP1, d);
1458                         M_CMOVUGT(REG_ITMP2, d);
1459                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1460                         emit_store_dst(jd, iptr, d);
1461                         break;
1462
1463                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1464                                           /* == => 0, < => 1, > => -1 */
1465
1466                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1467                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1468                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1469                         M_CLR(d);
1470                         M_MOV_IMM(1, REG_ITMP1);
1471                         M_MOV_IMM(-1, REG_ITMP2);
1472                         emit_ucomisd_reg_reg(cd, s1, s2);
1473                         M_CMOVULT(REG_ITMP1, d);
1474                         M_CMOVUGT(REG_ITMP2, d);
1475                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1476                         emit_store_dst(jd, iptr, d);
1477                         break;
1478
1479
1480                 /* memory operations **************************************************/
1481
1482                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1483
1484                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1485                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1486                         /* implicit null-pointer check */
1487                         M_ILD(d, s1, OFFSET(java_array_t, size));
1488                         emit_store_dst(jd, iptr, d);
1489                         break;
1490
1491                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1492
1493                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1494                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1495                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1496                         /* implicit null-pointer check */
1497                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1498                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1499                         emit_store_dst(jd, iptr, d);
1500                         break;
1501
1502                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1503
1504                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1505                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1506                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1507                         /* implicit null-pointer check */
1508                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1509                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1510                         emit_store_dst(jd, iptr, d);
1511                         break;
1512
1513                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1514
1515                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1516                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1517                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1518                         /* implicit null-pointer check */
1519                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1520                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1521                         emit_store_dst(jd, iptr, d);
1522                         break;
1523
1524                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1525
1526                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1527                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1528                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1529                         /* implicit null-pointer check */
1530                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1531                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1532                         emit_store_dst(jd, iptr, d);
1533                         break;
1534
1535                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1536
1537                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1538                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1539                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1540                         /* implicit null-pointer check */
1541                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1542                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1543                         emit_store_dst(jd, iptr, d);
1544                         break;
1545
1546                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1547
1548                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1549                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1550                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1551                         /* implicit null-pointer check */
1552                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1553                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1554                         emit_store_dst(jd, iptr, d);
1555                         break;
1556
1557                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1558
1559                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1560                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1561                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1562                         /* implicit null-pointer check */
1563                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1564                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1565                         emit_store_dst(jd, iptr, d);
1566                         break;
1567
1568                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1569
1570                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1571                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1572                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1573                         /* implicit null-pointer check */
1574                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1575                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1576                         emit_store_dst(jd, iptr, d);
1577                         break;
1578
1579
1580                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1581
1582                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1583                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1584                         /* implicit null-pointer check */
1585                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1586                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1587                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1588                         break;
1589
1590                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1591
1592                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1593                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1594                         /* implicit null-pointer check */
1595                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1596                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1597                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1598                         break;
1599
1600                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1601
1602                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1603                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1604                         /* implicit null-pointer check */
1605                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1606                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1607                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1608                         break;
1609
1610                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1611
1612                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1613                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1614                         /* implicit null-pointer check */
1615                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1616                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1617                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1618                         break;
1619
1620                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1621
1622                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1623                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1624                         /* implicit null-pointer check */
1625                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1626                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1627                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1628                         break;
1629
1630                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1631
1632                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1633                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1634                         /* implicit null-pointer check */
1635                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1636                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1637                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1638                         break;
1639
1640                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1641
1642                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1643                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1644                         /* implicit null-pointer check */
1645                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1646                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1647                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1648                         break;
1649
1650                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1651
1652                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1653                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1654                         /* implicit null-pointer check */
1655                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1656                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1657
1658                         M_MOV(s1, REG_A0);
1659                         M_MOV(s3, REG_A1);
1660                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
1661                         M_CALL(REG_ITMP1);
1662                         emit_arraystore_check(cd, iptr);
1663
1664                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1665                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1666                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1667                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1668                         break;
1669
1670
1671                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1672
1673                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1674                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1675                         /* implicit null-pointer check */
1676                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1677                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1678                         break;
1679
1680                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1681
1682                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1683                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1684                         /* implicit null-pointer check */
1685                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1686                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1687                         break;
1688
1689                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1690
1691                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1692                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1693                         /* implicit null-pointer check */
1694                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1695                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1696                         break;
1697
1698                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1699
1700                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1701                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1702                         /* implicit null-pointer check */
1703                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1704                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1705                         break;
1706
1707                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1708
1709                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1710                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1711                         /* implicit null-pointer check */
1712                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1713
1714                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1715                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1716                         }
1717                         else {
1718                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1719                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1720                         }
1721                         break;
1722
1723                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1724
1725                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1726                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1727                         /* implicit null-pointer check */
1728                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1729                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1730                         break;
1731
1732
1733                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1734
1735                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1736                                 uf        = iptr->sx.s23.s3.uf;
1737                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1738                                 disp      = dseg_add_unique_address(cd, uf);
1739
1740 /*                              PROFILE_CYCLE_STOP; */
1741
1742                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1743
1744 /*                              PROFILE_CYCLE_START; */
1745                         }
1746                         else {
1747                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1748                                 fieldtype = fi->type;
1749                                 disp      = dseg_add_address(cd, fi->value);
1750
1751                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1752                                         PROFILE_CYCLE_STOP;
1753
1754                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1755                                                                                   fi->clazz, 0);
1756
1757                                         PROFILE_CYCLE_START;
1758                                 }
1759                         }
1760
1761                         /* This approach is much faster than moving the field
1762                            address inline into a register. */
1763
1764                         M_ALD(REG_ITMP1, RIP, disp);
1765
1766                         switch (fieldtype) {
1767                         case TYPE_INT:
1768                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1769                                 M_ILD(d, REG_ITMP1, 0);
1770                                 break;
1771                         case TYPE_LNG:
1772                         case TYPE_ADR:
1773                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1774                                 M_LLD(d, REG_ITMP1, 0);
1775                                 break;
1776                         case TYPE_FLT:
1777                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1778                                 M_FLD(d, REG_ITMP1, 0);
1779                                 break;
1780                         case TYPE_DBL:
1781                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1782                                 M_DLD(d, REG_ITMP1, 0);
1783                                 break;
1784                         }
1785                         emit_store_dst(jd, iptr, d);
1786                         break;
1787
1788                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1789
1790                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1791                                 uf        = iptr->sx.s23.s3.uf;
1792                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1793                                 disp      = dseg_add_unique_address(cd, uf);
1794
1795 /*                              PROFILE_CYCLE_STOP; */
1796
1797                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1798
1799 /*                              PROFILE_CYCLE_START; */
1800                         }
1801                         else {
1802                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1803                                 fieldtype = fi->type;
1804                                 disp      = dseg_add_address(cd, fi->value);
1805
1806                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1807                                         PROFILE_CYCLE_STOP;
1808
1809                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1810                                                                                   fi->clazz, 0);
1811
1812                                         PROFILE_CYCLE_START;
1813                                 }
1814                         }
1815
1816                         /* This approach is much faster than moving the field
1817                            address inline into a register. */
1818
1819                         M_ALD(REG_ITMP1, RIP, disp);
1820
1821                         switch (fieldtype) {
1822                         case TYPE_INT:
1823                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1824                                 M_IST(s1, REG_ITMP1, 0);
1825                                 break;
1826                         case TYPE_LNG:
1827                         case TYPE_ADR:
1828                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1829                                 M_LST(s1, REG_ITMP1, 0);
1830                                 break;
1831                         case TYPE_FLT:
1832                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1833                                 M_FST(s1, REG_ITMP1, 0);
1834                                 break;
1835                         case TYPE_DBL:
1836                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1837                                 M_DST(s1, REG_ITMP1, 0);
1838                                 break;
1839                         }
1840                         break;
1841
1842                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1843                                           /* val = value (in current instruction)     */
1844                                           /* following NOP)                           */
1845
1846                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1847                                 uf        = iptr->sx.s23.s3.uf;
1848                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1849                                 disp      = dseg_add_unique_address(cd, uf);
1850
1851 /*                              PROFILE_CYCLE_STOP; */
1852
1853                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1854
1855 /*                              PROFILE_CYCLE_START; */
1856                         }
1857                         else {
1858                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1859                                 fieldtype = fi->type;
1860                                 disp      = dseg_add_address(cd, fi->value);
1861
1862                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1863                                         PROFILE_CYCLE_STOP;
1864
1865                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1866                                                                                   fi->clazz, 0);
1867
1868                                         PROFILE_CYCLE_START;
1869                                 }
1870                         }
1871
1872                         /* This approach is much faster than moving the field
1873                            address inline into a register. */
1874
1875                         M_ALD(REG_ITMP1, RIP, disp);
1876
1877                         switch (fieldtype) {
1878                         case TYPE_INT:
1879                         case TYPE_FLT:
1880                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1881                                 break;
1882                         case TYPE_LNG:
1883                         case TYPE_ADR:
1884                         case TYPE_DBL:
1885                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1886                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1887                                 else {
1888                                         M_MOV_IMM(iptr->sx.s23.s2.constval, REG_ITMP2);
1889                                         M_LST(REG_ITMP2, REG_ITMP1, 0);
1890                                 }
1891                                 break;
1892                         }
1893                         break;
1894
1895                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1896
1897                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1898
1899                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1900                                 uf        = iptr->sx.s23.s3.uf;
1901                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1902                                 disp      = 0;
1903
1904 /*                              PROFILE_CYCLE_STOP; */
1905
1906                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1907
1908 /*                              PROFILE_CYCLE_START; */
1909                         }
1910                         else {
1911                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1912                                 fieldtype = fi->type;
1913                                 disp      = fi->offset;
1914                         }
1915
1916                         /* implicit null-pointer check */
1917                         switch (fieldtype) {
1918                         case TYPE_INT:
1919                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1920                                 M_ILD32(d, s1, disp);
1921                                 break;
1922                         case TYPE_LNG:
1923                         case TYPE_ADR:
1924                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1925                                 M_LLD32(d, s1, disp);
1926                                 break;
1927                         case TYPE_FLT:
1928                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1929                                 M_FLD32(d, s1, disp);
1930                                 break;
1931                         case TYPE_DBL:
1932                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1933                                 M_DLD32(d, s1, disp);
1934                                 break;
1935                         }
1936                         emit_store_dst(jd, iptr, d);
1937                         break;
1938
1939                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1940
1941                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1942                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1943
1944                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1945                                 uf        = iptr->sx.s23.s3.uf;
1946                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1947                                 disp      = 0;
1948
1949 /*                              PROFILE_CYCLE_STOP; */
1950
1951                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1952
1953 /*                              PROFILE_CYCLE_START; */
1954                         }
1955                         else {
1956                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1957                                 fieldtype = fi->type;
1958                                 disp      = fi->offset;
1959                         }
1960
1961                         /* implicit null-pointer check */
1962                         switch (fieldtype) {
1963                         case TYPE_INT:
1964                                 M_IST32(s2, s1, disp);
1965                                 break;
1966                         case TYPE_LNG:
1967                         case TYPE_ADR:
1968                                 M_LST32(s2, s1, disp);
1969                                 break;
1970                         case TYPE_FLT:
1971                                 M_FST32(s2, s1, disp);
1972                                 break;
1973                         case TYPE_DBL:
1974                                 M_DST32(s2, s1, disp);
1975                                 break;
1976                         }
1977                         break;
1978
1979                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
1980                                           /* val = value (in current instruction)     */
1981                                           /* following NOP)                           */
1982
1983                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1984
1985                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1986                                 uf        = iptr->sx.s23.s3.uf;
1987                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1988                                 disp      = 0;
1989
1990 /*                              PROFILE_CYCLE_STOP; */
1991
1992                                 patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
1993
1994 /*                              PROFILE_CYCLE_START; */
1995                         }
1996                         else {
1997                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1998                                 fieldtype = fi->type;
1999                                 disp      = fi->offset;
2000                         }
2001
2002                         /* implicit null-pointer check */
2003                         switch (fieldtype) {
2004                         case TYPE_INT:
2005                         case TYPE_FLT:
2006                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2007                                 break;
2008                         case TYPE_LNG:
2009                         case TYPE_ADR:
2010                         case TYPE_DBL:
2011                                 /* XXX why no check for IS_IMM32? -- probably because of the patcher */
2012                                 M_MOV_IMM(iptr->sx.s23.s2.constval, REG_ITMP2);
2013                                 if (disp)  /* resolved, disp can never be 0 */
2014                                         M_LST(REG_ITMP2, s1, disp);
2015                                 else       /* unresolved */
2016                                         M_LST32(REG_ITMP2, s1, disp);
2017                                 break;
2018                         }
2019                         break;
2020
2021
2022                 /* branch operations **************************************************/
2023
2024                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2025
2026                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2027                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2028
2029                         PROFILE_CYCLE_STOP;
2030
2031 #ifdef ENABLE_VERIFIER
2032                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2033                                 uc = iptr->sx.s23.s2.uc;
2034
2035                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2036                         }
2037 #endif /* ENABLE_VERIFIER */
2038
2039                         M_CALL_IMM(0);                            /* passing exception pc */
2040                         M_POP(REG_ITMP2_XPC);
2041
2042                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2043                         M_JMP(REG_ITMP3);
2044                         break;
2045
2046                 case ICMD_GOTO:         /* ... ==> ...                                */
2047                 case ICMD_RET:
2048
2049                         emit_br(cd, iptr->dst.block);
2050                         ALIGNCODENOP;
2051                         break;
2052
2053                 case ICMD_JSR:          /* ... ==> ...                                */
2054
2055                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2056                         ALIGNCODENOP;
2057                         break;
2058
2059                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2060                 case ICMD_IFNONNULL:
2061
2062                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2063                         M_TEST(s1);
2064                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2065                         break;
2066
2067                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2068                 case ICMD_IFLT:
2069                 case ICMD_IFLE:
2070                 case ICMD_IFNE:
2071                 case ICMD_IFGT:
2072                 case ICMD_IFGE:
2073
2074                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2075                         M_ICMP_IMM(iptr->sx.val.i, s1);
2076                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2077                         break;
2078
2079                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2080                 case ICMD_IF_LNE:
2081                 case ICMD_IF_LLT:
2082                 case ICMD_IF_LGE:
2083                 case ICMD_IF_LGT:
2084                 case ICMD_IF_LLE:
2085
2086                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2087                         if (IS_IMM32(iptr->sx.val.l))
2088                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2089                         else {
2090                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2091                                 M_LCMP(REG_ITMP2, s1);
2092                         }
2093                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2094                         break;
2095
2096                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2097                 case ICMD_IF_ICMPNE:
2098                 case ICMD_IF_ICMPLT:
2099                 case ICMD_IF_ICMPGE:
2100                 case ICMD_IF_ICMPGT:
2101                 case ICMD_IF_ICMPLE:
2102
2103                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2104                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2105                         M_ICMP(s2, s1);
2106                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2107                         break;
2108
2109                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2110                 case ICMD_IF_ACMPNE:
2111
2112                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2113                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2114                         M_LCMP(s2, s1);
2115                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2116                         break;
2117
2118                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2119                 case ICMD_IF_LCMPNE:
2120                 case ICMD_IF_LCMPLT:
2121                 case ICMD_IF_LCMPGE:
2122                 case ICMD_IF_LCMPGT:
2123                 case ICMD_IF_LCMPLE:
2124
2125                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2126                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2127                         M_LCMP(s2, s1);
2128                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2129                         break;
2130
2131                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2132                 case ICMD_LRETURN:
2133
2134                         REPLACEMENT_POINT_RETURN(cd, iptr);
2135                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2136                         M_INTMOVE(s1, REG_RESULT);
2137                         goto nowperformreturn;
2138
2139                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2140
2141                         REPLACEMENT_POINT_RETURN(cd, iptr);
2142                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2143                         M_INTMOVE(s1, REG_RESULT);
2144
2145 #ifdef ENABLE_VERIFIER
2146                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2147                                 uc = iptr->sx.s23.s2.uc;
2148
2149                                 PROFILE_CYCLE_STOP;
2150
2151                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2152
2153                                 PROFILE_CYCLE_START;
2154                         }
2155 #endif /* ENABLE_VERIFIER */
2156                         goto nowperformreturn;
2157
2158                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2159                 case ICMD_DRETURN:
2160
2161                         REPLACEMENT_POINT_RETURN(cd, iptr);
2162                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2163                         M_FLTMOVE(s1, REG_FRESULT);
2164                         goto nowperformreturn;
2165
2166                 case ICMD_RETURN:      /* ...  ==> ...                                */
2167
2168                         REPLACEMENT_POINT_RETURN(cd, iptr);
2169
2170 nowperformreturn:
2171                         {
2172                         s4 i, p;
2173
2174                         p = cd->stackframesize;
2175
2176 #if !defined(NDEBUG)
2177                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2178                                 emit_verbosecall_exit(jd);
2179 #endif /* !defined(NDEBUG) */
2180
2181 #if defined(ENABLE_THREADS)
2182                         if (checksync && code_is_synchronized(code)) {
2183                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2184
2185                                 /* we need to save the proper return value */
2186                                 switch (iptr->opc) {
2187                                 case ICMD_IRETURN:
2188                                 case ICMD_ARETURN:
2189                                 case ICMD_LRETURN:
2190                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2191                                         break;
2192                                 case ICMD_FRETURN:
2193                                 case ICMD_DRETURN:
2194                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2195                                         break;
2196                                 }
2197
2198                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2199                                 M_CALL(REG_ITMP1);
2200
2201                                 /* and now restore the proper return value */
2202                                 switch (iptr->opc) {
2203                                 case ICMD_IRETURN:
2204                                 case ICMD_ARETURN:
2205                                 case ICMD_LRETURN:
2206                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2207                                         break;
2208                                 case ICMD_FRETURN:
2209                                 case ICMD_DRETURN:
2210                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2211                                         break;
2212                                 }
2213                         }
2214 #endif
2215
2216                         /* restore saved registers */
2217
2218                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2219                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2220                         }
2221                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2222                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2223                         }
2224
2225                         /* deallocate stack */
2226
2227                         if (cd->stackframesize)
2228                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2229
2230                         /* generate method profiling code */
2231
2232                         PROFILE_CYCLE_STOP;
2233
2234                         M_RET;
2235                         }
2236                         break;
2237
2238
2239                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2240                         {
2241                                 s4 i, l;
2242                                 branch_target_t *table;
2243
2244                                 table = iptr->dst.table;
2245
2246                                 l = iptr->sx.s23.s2.tablelow;
2247                                 i = iptr->sx.s23.s3.tablehigh;
2248
2249                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2250                                 M_INTMOVE(s1, REG_ITMP1);
2251
2252                                 if (l != 0)
2253                                         M_ISUB_IMM(l, REG_ITMP1);
2254
2255                                 /* number of targets */
2256                                 i = i - l + 1;
2257
2258                 /* range check */
2259
2260                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2261                                 emit_bugt(cd, table[0].block);
2262
2263                                 /* build jump table top down and use address of lowest entry */
2264
2265                                 table += i;
2266
2267                                 while (--i >= 0) {
2268                                         dseg_add_target(cd, table->block);
2269                                         --table;
2270                                 }
2271
2272                                 /* length of dataseg after last dseg_add_target is used
2273                                    by load */
2274
2275                                 M_MOV_IMM(0, REG_ITMP2);
2276                                 dseg_adddata(cd);
2277                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2278                                 M_JMP(REG_ITMP1);
2279                         }
2280                         break;
2281
2282
2283                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2284                         {
2285                                 s4 i;
2286                                 lookup_target_t *lookup;
2287
2288                                 lookup = iptr->dst.lookup;
2289
2290                                 i = iptr->sx.s23.s2.lookupcount;
2291
2292                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2293                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2294
2295                                 while (--i >= 0) {
2296                                         M_ICMP_IMM(lookup->value, s1);
2297                                         emit_beq(cd, lookup->target.block);
2298                                         lookup++;
2299                                 }
2300
2301                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2302                                 ALIGNCODENOP;
2303                         }
2304                         break;
2305
2306
2307                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2308
2309                         REPLACEMENT_POINT_FORGC_BUILTIN(cd, iptr);
2310
2311                         bte = iptr->sx.s23.s3.bte;
2312                         md  = bte->md;
2313                         goto gen_method;
2314
2315                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2316
2317                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2318                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2319                 case ICMD_INVOKEINTERFACE:
2320
2321                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2322
2323                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2324                                 lm = NULL;
2325                                 um = iptr->sx.s23.s3.um;
2326                                 md = um->methodref->parseddesc.md;
2327                         }
2328                         else {
2329                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2330                                 um = NULL;
2331                                 md = lm->parseddesc;
2332                         }
2333
2334 gen_method:
2335                         s3 = md->paramcount;
2336
2337                         MCODECHECK((20 * s3) + 128);
2338
2339                         /* copy arguments to registers or stack location */
2340
2341                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2342                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2343                                 d   = md->params[s3].regoff;
2344
2345                                 /* already preallocated (ARGVAR)? */
2346
2347                                 if (var->flags & PREALLOC)
2348                                         continue;
2349
2350                                 if (IS_INT_LNG_TYPE(var->type)) {
2351                                         if (!md->params[s3].inmemory) {
2352                                                 s1 = emit_load(jd, iptr, var, d);
2353                                                 M_INTMOVE(s1, d);
2354                                         }
2355                                         else {
2356                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2357                                                 M_LST(s1, REG_SP, d);
2358                                         }
2359                                 }
2360                                 else {
2361                                         if (!md->params[s3].inmemory) {
2362                                                 s1 = emit_load(jd, iptr, var, d);
2363                                                 M_FLTMOVE(s1, d);
2364                                         }
2365                                         else {
2366                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2367
2368                                                 if (IS_2_WORD_TYPE(var->type))
2369                                                         M_DST(s1, REG_SP, d);
2370                                                 else
2371                                                         M_FST(s1, REG_SP, d);
2372                                         }
2373                                 }
2374                         }
2375
2376                         /* generate method profiling code */
2377
2378                         PROFILE_CYCLE_STOP;
2379
2380                         switch (iptr->opc) {
2381                         case ICMD_BUILTIN:
2382                                 if (bte->stub == NULL) {
2383                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2384                                 }
2385                                 else {
2386                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2387                                 }
2388                                 M_CALL(REG_ITMP1);
2389                                 break;
2390
2391                         case ICMD_INVOKESPECIAL:
2392                                 emit_nullpointer_check(cd, iptr, REG_A0);
2393                                 /* fall through */
2394
2395                         case ICMD_INVOKESTATIC:
2396                                 if (lm == NULL) {
2397                                         disp = dseg_add_unique_address(cd, um);
2398
2399                                         patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
2400                                                                                   um, disp);
2401                                 }
2402                                 else {
2403                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2404                                 }
2405
2406                                 M_ALD(REG_ITMP2, RIP, disp);
2407                                 M_CALL(REG_ITMP2);
2408                                 break;
2409
2410                         case ICMD_INVOKEVIRTUAL:
2411                                 if (lm == NULL) {
2412                                         patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
2413
2414                                         s1 = 0;
2415                                 }
2416                                 else {
2417                                         s1 = OFFSET(vftbl_t, table[0]) +
2418                                                 sizeof(methodptr) * lm->vftblindex;
2419                                 }
2420
2421                                 /* implicit null-pointer check */
2422                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2423                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2424                                 M_CALL(REG_ITMP3);
2425                                 break;
2426
2427                         case ICMD_INVOKEINTERFACE:
2428                                 if (lm == NULL) {
2429                                         patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
2430
2431                                         s1 = 0;
2432                                         s2 = 0;
2433                                 }
2434                                 else {
2435                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2436                                                 sizeof(methodptr) * lm->clazz->index;
2437
2438                                         s2 = sizeof(methodptr) * (lm - lm->clazz->methods);
2439                                 }
2440
2441                                 /* implicit null-pointer check */
2442                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2443                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2444                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2445                                 M_CALL(REG_ITMP3);
2446                                 break;
2447                         }
2448
2449                         /* generate method profiling code */
2450
2451                         PROFILE_CYCLE_START;
2452
2453                         /* store size of call code in replacement point */
2454
2455                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2456                         REPLACEMENT_POINT_FORGC_BUILTIN_RETURN(cd, iptr);
2457
2458                         /* store return value */
2459
2460                         switch (md->returntype.type) {
2461                         case TYPE_INT:
2462                         case TYPE_LNG:
2463                         case TYPE_ADR:
2464                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2465                                 M_INTMOVE(REG_RESULT, s1);
2466                                 emit_store_dst(jd, iptr, s1);
2467                                 break;
2468                         case TYPE_FLT:
2469                         case TYPE_DBL:
2470                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2471                                 M_FLTMOVE(REG_FRESULT, s1);
2472                                 emit_store_dst(jd, iptr, s1);
2473                                 break;
2474                         default:
2475                                 /* TYPE_VOID */
2476                                 break;
2477                         }
2478                         break;
2479
2480
2481                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2482
2483                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2484                                 /* object type cast-check */
2485
2486                                 classinfo *super;
2487                                 s4         superindex;
2488
2489                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2490                                         super      = NULL;
2491                                         superindex = 0;
2492                                 }
2493                                 else {
2494                                         super      = iptr->sx.s23.s3.c.cls;
2495                                         superindex = super->index;
2496                                 }
2497
2498                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2499
2500                                 /* if class is not resolved, check which code to call */
2501
2502                                 if (super == NULL) {
2503                                         M_TEST(s1);
2504                                         emit_label_beq(cd, BRANCH_LABEL_1);
2505
2506                                         patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2507                                                                                   iptr->sx.s23.s3.c.ref, 0);
2508
2509                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2510                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2511                                         emit_label_beq(cd, BRANCH_LABEL_2);
2512                                 }
2513
2514                                 /* interface checkcast code */
2515
2516                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2517                                         if (super != NULL) {
2518                                                 M_TEST(s1);
2519                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2520                                         }
2521
2522                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2523
2524                                         if (super == NULL) {
2525                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
2526                                                                                           iptr->sx.s23.s3.c.ref,
2527                                                                                           0);
2528                                         }
2529
2530                                         M_ILD32(REG_ITMP3,
2531                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2532                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2533                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2534
2535                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2536                                                         OFFSET(vftbl_t, interfacetable[0]) -
2537                                                         superindex * sizeof(methodptr*));
2538                                         M_TEST(REG_ITMP3);
2539                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2540
2541                                         if (super == NULL)
2542                                                 emit_label_br(cd, BRANCH_LABEL_4);
2543                                         else
2544                                                 emit_label(cd, BRANCH_LABEL_3);
2545                                 }
2546
2547                                 /* class checkcast code */
2548
2549                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2550                                         if (super == NULL) {
2551                                                 emit_label(cd, BRANCH_LABEL_2);
2552
2553                                                 cr   = iptr->sx.s23.s3.c.ref;
2554                                                 disp = dseg_add_unique_address(cd, cr);
2555
2556                                                 patcher_add_patch_ref(jd,
2557                                                                                           PATCHER_resolve_classref_to_vftbl,
2558                                                                                           cr, disp);
2559                                         }
2560                                         else {
2561                                                 M_TEST(s1);
2562                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2563
2564                                                 disp = dseg_add_address(cd, super->vftbl);
2565                                         }
2566
2567                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2568                                         M_ALD(REG_ITMP3, RIP, disp);
2569
2570                                         M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2571
2572                                         /*                                      if (s1 != REG_ITMP1) { */
2573                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2574                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2575                                         /*                                                                                              REG_ITMP1); */
2576                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2577                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2578                                         /*                                                                                              REG_ITMP3); */
2579                                         /*  #if defined(ENABLE_THREADS) */
2580                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2581                                         /*  #endif */
2582                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2583
2584                                         /*                                      } else { */
2585
2586                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
2587                                         M_ISUB(REG_ITMP3, REG_ITMP2);
2588                                         M_ALD(REG_ITMP3, RIP, disp);
2589                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2590                                         /*                                      } */
2591
2592                                         M_ICMP(REG_ITMP3, REG_ITMP2);
2593                                         emit_classcast_check(cd, iptr, BRANCH_UGT, REG_ITMP3, s1);
2594
2595                                         if (super != NULL)
2596                                                 emit_label(cd, BRANCH_LABEL_5);
2597                                 }
2598
2599                                 if (super == NULL) {
2600                                         emit_label(cd, BRANCH_LABEL_1);
2601                                         emit_label(cd, BRANCH_LABEL_4);
2602                                 }
2603
2604                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2605                         }
2606                         else {
2607                                 /* array type cast-check */
2608
2609                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2610                                 M_INTMOVE(s1, REG_A0);
2611
2612                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2613                                         cr   = iptr->sx.s23.s3.c.ref;
2614                                         disp = dseg_add_unique_address(cd, cr);
2615
2616                                         patcher_add_patch_ref(jd,
2617                                                                                   PATCHER_resolve_classref_to_classinfo,
2618                                                                                   cr, disp);
2619                                 }
2620                                 else {
2621                                         disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2622                                 }
2623
2624                                 M_ALD(REG_A1, RIP, disp);
2625                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2626                                 M_CALL(REG_ITMP1);
2627
2628                                 /* s1 may have been destroyed over the function call */
2629                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2630                                 M_TEST(REG_RESULT);
2631                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2632
2633                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2634                         }
2635
2636                         M_INTMOVE(s1, d);
2637                         emit_store_dst(jd, iptr, d);
2638                         break;
2639
2640                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2641
2642                         {
2643                         classinfo *super;
2644                         s4         superindex;
2645
2646                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2647                                 super      = NULL;
2648                                 superindex = 0;
2649                         }
2650                         else {
2651                                 super      = iptr->sx.s23.s3.c.cls;
2652                                 superindex = super->index;
2653                         }
2654
2655                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2656                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2657
2658                         if (s1 == d) {
2659                                 M_INTMOVE(s1, REG_ITMP1);
2660                                 s1 = REG_ITMP1;
2661                         }
2662
2663                         M_CLR(d);
2664
2665                         /* if class is not resolved, check which code to call */
2666
2667                         if (super == NULL) {
2668                                 M_TEST(s1);
2669                                 emit_label_beq(cd, BRANCH_LABEL_1);
2670
2671                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2672                                                                           iptr->sx.s23.s3.c.ref, 0);
2673
2674                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2675                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2676                                 emit_label_beq(cd, BRANCH_LABEL_2);
2677                         }
2678
2679                         /* interface instanceof code */
2680
2681                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2682                                 if (super != NULL) {
2683                                         M_TEST(s1);
2684                                         emit_label_beq(cd, BRANCH_LABEL_3);
2685                                 }
2686
2687                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2688
2689                                 if (super == NULL) {
2690                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
2691                                                                                   iptr->sx.s23.s3.c.ref, 0);
2692                                 }
2693
2694                                 M_ILD32(REG_ITMP3,
2695                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2696                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2697
2698                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2699
2700                                 M_BLE(a);
2701                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2702                                                 OFFSET(vftbl_t, interfacetable[0]) -
2703                                                 superindex * sizeof(methodptr*));
2704                                 M_TEST(REG_ITMP1);
2705                                 M_SETNE(d);
2706
2707                                 if (super == NULL)
2708                                         emit_label_br(cd, BRANCH_LABEL_4);
2709                                 else
2710                                         emit_label(cd, BRANCH_LABEL_3);
2711                         }
2712
2713                         /* class instanceof code */
2714
2715                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2716                                 if (super == NULL) {
2717                                         emit_label(cd, BRANCH_LABEL_2);
2718
2719                                         cr   = iptr->sx.s23.s3.c.ref;
2720                                         disp = dseg_add_unique_address(cd, cr);
2721
2722                                         patcher_add_patch_ref(jd,
2723                                                                                   PATCHER_resolve_classref_to_vftbl,
2724                                                                                   cr, disp);
2725                                 }
2726                                 else {
2727                                         M_TEST(s1);
2728                                         emit_label_beq(cd, BRANCH_LABEL_5);
2729
2730                                         disp = dseg_add_address(cd, super->vftbl);
2731                                 }
2732
2733                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2734                                 M_ALD(REG_ITMP2, RIP, disp);
2735
2736                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
2737                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2738                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2739
2740                                 M_ISUB(REG_ITMP2, REG_ITMP1);
2741                                 M_CLR(d); /* may be REG_ITMP2 */
2742                                 M_ICMP(REG_ITMP3, REG_ITMP1);
2743                                 M_SETULE(d);
2744
2745                                 if (super != NULL)
2746                                         emit_label(cd, BRANCH_LABEL_5);
2747                         }
2748
2749                         if (super == NULL) {
2750                                 emit_label(cd, BRANCH_LABEL_1);
2751                                 emit_label(cd, BRANCH_LABEL_4);
2752                         }
2753
2754                         emit_store_dst(jd, iptr, d);
2755                         }
2756                         break;
2757
2758                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2759
2760                         /* check for negative sizes and copy sizes to stack if necessary  */
2761
2762                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2763
2764                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2765
2766                                 /* copy SAVEDVAR sizes to stack */
2767                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2768
2769                                 /* Already Preallocated? */
2770                                 if (!(var->flags & PREALLOC)) {
2771                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2772                                         M_LST(s2, REG_SP, s1 * 8);
2773                                 }
2774                         }
2775
2776                         /* a0 = dimension count */
2777
2778                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2779
2780                         /* is a patcher function set? */
2781
2782                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2783                                 cr   = iptr->sx.s23.s3.c.ref;
2784                                 disp = dseg_add_unique_address(cd, cr);
2785
2786                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
2787                                                                           cr, disp);
2788                         }
2789                         else {
2790                                 disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2791                         }
2792
2793                         /* a1 = classinfo */
2794
2795                         M_ALD(REG_A1, RIP, disp);
2796
2797                         /* a2 = pointer to dimensions = stack pointer */
2798
2799                         M_MOV(REG_SP, REG_A2);
2800
2801                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2802                         M_CALL(REG_ITMP1);
2803
2804                         /* check for exception before result assignment */
2805
2806                         emit_exception_check(cd, iptr);
2807
2808                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2809                         M_INTMOVE(REG_RESULT, s1);
2810                         emit_store_dst(jd, iptr, s1);
2811                         break;
2812
2813                 default:
2814                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2815                                                                                    iptr->opc);
2816                         return false;
2817         } /* switch */
2818
2819         } /* for instruction */
2820
2821         MCODECHECK(512); /* XXX require a lower number? */
2822
2823         /* At the end of a basic block we may have to append some nops,
2824            because the patcher stub calling code might be longer than the
2825            actual instruction. So codepatching does not change the
2826            following block unintentionally. */
2827
2828         if (cd->mcodeptr < cd->lastmcodeptr) {
2829                 while (cd->mcodeptr < cd->lastmcodeptr) {
2830                         M_NOP;
2831                 }
2832         }
2833
2834         } /* if (bptr -> flags >= BBREACHED) */
2835         } /* for basic block */
2836
2837         /* Generate patcher traps. */
2838
2839         emit_patcher_traps(jd);
2840
2841         /* everything's ok */
2842
2843         return true;
2844 }
2845
2846
2847 /* codegen_emit_stub_native ****************************************************
2848
2849    Emits a stub routine which calls a native method.
2850
2851 *******************************************************************************/
2852
2853 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f, int skipparams)
2854 {
2855         methodinfo  *m;
2856         codeinfo    *code;
2857         codegendata *cd;
2858         methoddesc  *md;
2859         int          i, j;
2860         int          s1, s2;
2861         int          disp;
2862
2863         /* Sanity check. */
2864
2865         assert(f != NULL);
2866
2867         /* Get required compiler data. */
2868
2869         m    = jd->m;
2870         code = jd->code;
2871         cd   = jd->cd;
2872
2873         /* initialize variables */
2874
2875         md = m->parseddesc;
2876
2877         /* calculate stack frame size */
2878
2879         cd->stackframesize =
2880                 sizeof(stackframeinfo_t) / SIZEOF_VOID_P +
2881                 sizeof(localref_table) / SIZEOF_VOID_P +
2882                 md->paramcount +
2883                 (md->returntype.type == TYPE_VOID ? 0 : 1) +
2884                 nmd->memuse;
2885
2886         ALIGN_ODD(cd->stackframesize);              /* keep stack 16-byte aligned */
2887
2888         /* create method header */
2889
2890         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2891         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2892         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2893         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2894         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2895
2896 #if defined(ENABLE_PROFILING)
2897         /* generate native method profiling code */
2898
2899         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
2900                 /* count frequency */
2901
2902                 M_MOV_IMM(code, REG_ITMP3);
2903                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
2904         }
2905 #endif
2906
2907         /* generate stub code */
2908
2909         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2910
2911 #if defined(ENABLE_GC_CACAO)
2912         /* Save callee saved integer registers in stackframeinfo (GC may
2913            need to recover them during a collection). */
2914
2915         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
2916                 OFFSET(stackframeinfo_t, intregs);
2917
2918         for (i = 0; i < INT_SAV_CNT; i++)
2919                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2920 #endif
2921
2922         /* save integer and float argument registers */
2923
2924         for (i = 0; i < md->paramcount; i++) {
2925                 if (!md->params[i].inmemory) {
2926                         s1 = md->params[i].regoff;
2927
2928                         switch (md->paramtypes[i].type) {
2929                         case TYPE_INT:
2930                         case TYPE_LNG:
2931                         case TYPE_ADR:
2932                                 M_LST(s1, REG_SP, i * 8);
2933                                 break;
2934                         case TYPE_FLT:
2935                         case TYPE_DBL:
2936                                 M_DST(s1, REG_SP, i * 8);
2937                                 break;
2938                         }
2939                 }
2940         }
2941
2942         /* create dynamic stack info */
2943
2944         M_MOV(REG_SP, REG_A0);
2945         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
2946         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
2947         M_CALL(REG_ITMP1);
2948
2949         /* remember class argument */
2950
2951         if (m->flags & ACC_STATIC)
2952                 M_MOV(REG_RESULT, REG_ITMP2);
2953
2954         /* restore integer and float argument registers */
2955
2956         for (i = 0; i < md->paramcount; i++) {
2957                 if (!md->params[i].inmemory) {
2958                         s1 = md->params[i].regoff;
2959
2960                         switch (md->paramtypes[i].type) {
2961                         case TYPE_INT:
2962                         case TYPE_LNG:
2963                         case TYPE_ADR:
2964                                 M_LLD(s1, REG_SP, i * 8);
2965                                 break;
2966                         case TYPE_FLT:
2967                         case TYPE_DBL:
2968                                 M_DLD(s1, REG_SP, i * 8);
2969                                 break;
2970                         }
2971                 }
2972         }
2973
2974         /* Copy or spill arguments to new locations. */
2975
2976         for (i = md->paramcount - 1, j = i + skipparams; i >= 0; i--, j--) {
2977                 s2 = nmd->params[j].regoff;
2978
2979                 switch (md->paramtypes[i].type) {
2980                 case TYPE_INT:
2981                 case TYPE_LNG:
2982                 case TYPE_ADR:
2983                         if (!md->params[i].inmemory) {
2984                                 s1 = md->params[i].regoff;
2985
2986                                 if (!nmd->params[j].inmemory)
2987                                         M_INTMOVE(s1, s2);
2988                                 else
2989                                         M_LST(s1, REG_SP, s2);
2990                         }
2991                         else {
2992                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
2993                                 M_LLD(REG_ITMP1, REG_SP, s1);
2994                                 M_LST(REG_ITMP1, REG_SP, s2);
2995                         }
2996                         break;
2997                 case TYPE_FLT:
2998                         /* We only copy spilled float arguments, as the float
2999                            argument registers keep unchanged. */
3000
3001                         if (md->params[i].inmemory) {
3002                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3003
3004                                 M_FLD(REG_FTMP1, REG_SP, s1);
3005                                 M_FST(REG_FTMP1, REG_SP, s2);
3006                         }
3007                         break;
3008                 case TYPE_DBL:
3009                         if (md->params[i].inmemory) {
3010                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3011                                 M_DLD(REG_FTMP1, REG_SP, s1);
3012                                 M_DST(REG_FTMP1, REG_SP, s2);
3013                         }
3014                         break;
3015                 }
3016         }
3017
3018         /* Handle native Java methods. */
3019
3020         if (m->flags & ACC_NATIVE) {
3021                 /* put class into second argument register */
3022
3023                 if (m->flags & ACC_STATIC)
3024                         M_MOV(REG_ITMP2, REG_A1);
3025
3026                 /* put env into first argument register */
3027
3028                 M_MOV_IMM(VM_get_jnienv(), REG_A0);
3029         }
3030
3031         /* Call the native function. */
3032
3033         disp = dseg_add_functionptr(cd, f);
3034         M_ALD(REG_ITMP1, RIP, disp);
3035         M_CALL(REG_ITMP1);
3036
3037         /* save return value */
3038
3039         switch (md->returntype.type) {
3040         case TYPE_INT:
3041         case TYPE_LNG:
3042         case TYPE_ADR:
3043                 switch (md->returntype.primitivetype) {
3044                 case PRIMITIVETYPE_BOOLEAN:
3045                         M_BZEXT(REG_RESULT, REG_RESULT);
3046                         break;
3047                 case PRIMITIVETYPE_BYTE:
3048                         M_BSEXT(REG_RESULT, REG_RESULT);
3049                         break;
3050                 case PRIMITIVETYPE_CHAR:
3051                         M_CZEXT(REG_RESULT, REG_RESULT);
3052                         break;
3053                 case PRIMITIVETYPE_SHORT:
3054                         M_SSEXT(REG_RESULT, REG_RESULT);
3055                         break;
3056                 }
3057                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3058                 break;
3059         case TYPE_FLT:
3060         case TYPE_DBL:
3061                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3062                 break;
3063         case TYPE_VOID:
3064                 break;
3065         }
3066
3067         /* remove native stackframe info */
3068
3069         M_MOV(REG_SP, REG_A0);
3070         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3071         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3072         M_CALL(REG_ITMP1);
3073         M_MOV(REG_RESULT, REG_ITMP3);
3074
3075         /* restore return value */
3076
3077         switch (md->returntype.type) {
3078         case TYPE_INT:
3079         case TYPE_LNG:
3080         case TYPE_ADR:
3081                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3082                 break;
3083         case TYPE_FLT:
3084         case TYPE_DBL:
3085                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3086                 break;
3087         case TYPE_VOID:
3088                 break;
3089         }
3090
3091 #if defined(ENABLE_GC_CACAO)
3092         /* Restore callee saved integer registers from stackframeinfo (GC
3093            might have modified them during a collection). */
3094
3095         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
3096                 OFFSET(stackframeinfo_t, intregs);
3097
3098         for (i = 0; i < INT_SAV_CNT; i++)
3099                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3100 #endif
3101
3102         /* remove stackframe */
3103
3104         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3105
3106         /* test for exception */
3107
3108         M_TEST(REG_ITMP3);
3109         M_BNE(1);
3110         M_RET;
3111
3112         /* handle exception */
3113
3114         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3115         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3116         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3117
3118         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3119         M_JMP(REG_ITMP3);
3120 }
3121
3122
3123 /*
3124  * These are local overrides for various environment variables in Emacs.
3125  * Please do not remove this and leave it at the end of the file, where
3126  * Emacs will automagically detect them.
3127  * ---------------------------------------------------------------------
3128  * Local variables:
3129  * mode: c
3130  * indent-tabs-mode: t
3131  * c-basic-offset: 4
3132  * tab-width: 4
3133  * End:
3134  * vim:noexpandtab:sw=4:ts=4:
3135  */