src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007, 2008
   4    CACAOVM - Verein zur Foerderung der freien virtuellen Maschine CACAO
   5
   6    This file is part of CACAO.
   7
   8    This program is free software; you can redistribute it and/or
   9    modify it under the terms of the GNU General Public License as
  10    published by the Free Software Foundation; either version 2, or (at
  11    your option) any later version.
  12
  13    This program is distributed in the hope that it will be useful, but
  14    WITHOUT ANY WARRANTY; without even the implied warranty of
  15    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16    General Public License for more details.
  17
  18    You should have received a copy of the GNU General Public License
  19    along with this program; if not, write to the Free Software
  20    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  21    02110-1301, USA.
  22
  23 */
  24
  25
  26 #include "config.h"
  27
  28 #include <assert.h>
  29 #include <stdio.h>
  30 #include <stdint.h>
  31
  32 #include "vm/types.h"
  33
  34 #include "md-abi.h"
  35
  36 #include "vm/jit/x86_64/arch.h"
  37 #include "vm/jit/x86_64/codegen.h"
  38 #include "vm/jit/x86_64/emit.h"
  39
  40 #include "mm/memory.h"
  41
  42 #include "native/localref.h"
  43 #include "native/native.h"
  44
  45 #include "threads/lock-common.h"
  46
  47 #include "vm/jit/builtin.hpp"
  48 #include "vm/exceptions.hpp"
  49 #include "vm/global.h"
  50 #include "vm/loader.hpp"
  51 #include "vm/options.h"
  52 #include "vm/primitive.hpp"
  53 #include "vm/statistics.h"
  54 #include "vm/string.hpp"
  55 #include "vm/vm.hpp"
  56
  57 #include "vm/jit/abi.h"
  58 #include "vm/jit/asmpart.h"
  59 #include "vm/jit/code.hpp"
  60 #include "vm/jit/codegen-common.hpp"
  61 #include "vm/jit/dseg.h"
  62 #include "vm/jit/emit-common.hpp"
  63 #include "vm/jit/jit.hpp"
  64 #include "vm/jit/linenumbertable.h"
  65 #include "vm/jit/methodheader.h"
  66 #include "vm/jit/parse.h"
  67 #include "vm/jit/patcher-common.hpp"
  68 #include "vm/jit/reg.h"
  69 #include "vm/jit/replace.hpp"
  70 #include "vm/jit/stacktrace.hpp"
  71 #include "vm/jit/trap.h"
  72
  73 #if defined(ENABLE_LSRA)
  74 # include "vm/jit/allocator/lsra.h"
  75 #endif
  76
  77
  78 /* codegen_emit ****************************************************************
  79
  80    Generates machine code.
  81
  82 *******************************************************************************/
  83
  84 bool codegen_emit(jitdata *jd)
  85 {
  86         methodinfo         *m;
  87         codeinfo           *code;
  88         codegendata        *cd;
  89         registerdata       *rd;
  90         s4                  len, s1, s2, s3, d, disp;
  91         u2                  currentline;
  92         ptrint              a;
  93         varinfo            *var, *dst;
  94         basicblock         *bptr;
  95         instruction        *iptr;
  96         constant_classref  *cr;
  97         unresolved_class   *uc;
  98         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
  99         unresolved_method  *um;
 100         builtintable_entry *bte;
 101         methoddesc         *md;
 102         fieldinfo          *fi;
 103         unresolved_field   *uf;
 104         s4                  fieldtype;
 105         s4                 varindex;
 106
 107         /* get required compiler data */
 108
 109         m    = jd->m;
 110         code = jd->code;
 111         cd   = jd->cd;
 112         rd   = jd->rd;
 113
 114         /* prevent compiler warnings */
 115
 116         d   = 0;
 117         lm  = NULL;
 118         um  = NULL;
 119         bte = NULL;
 120
 121         {
 122         s4 i, p, t, l;
 123         s4 savedregs_num;
 124
 125         savedregs_num = 0;
 126
 127         /* space to save used callee saved registers */
 128
 129         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 130         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 131
 132         cd->stackframesize = rd->memuse + savedregs_num;
 133
 134 #if defined(ENABLE_THREADS)
 135         /* space to save argument of monitor_enter */
 136
 137         if (checksync && code_is_synchronized(code))
 138                 cd->stackframesize++;
 139 #endif
 140
 141         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 142            native code e.g. libc or jni (alignment problems with
 143            movaps). */
 144
 145         if (!code_is_leafmethod(code) || opt_verbosecall)
 146                 cd->stackframesize |= 0x1;
 147
 148         /* create method header */
 149
 150         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 151         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 152
 153         code->synchronizedoffset = rd->memuse * 8;
 154
 155         if (code_is_leafmethod(code))
 156                 (void) dseg_add_unique_s4(cd, 1);                  /* IsLeaf          */
 157         else
 158                 (void) dseg_add_unique_s4(cd, 0);                  /* IsLeaf          */
 159
 160         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 161         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 162
 163 #if defined(ENABLE_PROFILING)
 164         /* generate method profiling code */
 165
 166         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 167                 /* count frequency */
 168
 169                 M_MOV_IMM(code, REG_ITMP3);
 170                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 171
 172                 PROFILE_CYCLE_START;
 173         }
 174 #endif
 175
 176         /* create stack frame (if necessary) */
 177
 178         if (cd->stackframesize)
 179                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 180
 181         /* save used callee saved registers */
 182
 183         p = cd->stackframesize;
 184         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 185                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 186         }
 187         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 188                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 189         }
 190
 191         /* take arguments out of register or stack frame */
 192
 193         md = m->parseddesc;
 194
 195         for (p = 0, l = 0; p < md->paramcount; p++) {
 196                 t = md->paramtypes[p].type;
 197
 198                 varindex = jd->local_map[l * 5 + t];
 199
 200                 l++;
 201                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 202                         l++;
 203
 204                 if (varindex == UNUSED)
 205                         continue;
 206
 207                 var = VAR(varindex);
 208
 209                 s1 = md->params[p].regoff;
 210
 211                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 212                         if (!md->params[p].inmemory) {           /* register arguments    */
 213                                 if (!IS_INMEMORY(var->flags))
 214                                         M_INTMOVE(s1, var->vv.regoff);
 215                                 else
 216                                     M_LST(s1, REG_SP, var->vv.regoff);
 217                         }
 218                         else {                                 /* stack arguments       */
 219                                 if (!IS_INMEMORY(var->flags))
 220                                         /* + 8 for return address */
 221                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 222                                 else
 223                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 224                         }
 225                 }
 226                 else {                                     /* floating args         */
 227                         if (!md->params[p].inmemory) {           /* register arguments    */
 228                                 if (!IS_INMEMORY(var->flags))
 229                                         M_FLTMOVE(s1, var->vv.regoff);
 230                                 else
 231                                         M_DST(s1, REG_SP, var->vv.regoff);
 232                         }
 233                         else {                                 /* stack arguments       */
 234                                 if (!IS_INMEMORY(var->flags))
 235                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 236                                 else
 237                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 238                         }
 239                 }
 240         }
 241
 242         /* save monitorenter argument */
 243
 244 #if defined(ENABLE_THREADS)
 245         if (checksync && code_is_synchronized(code)) {
 246                 /* stack offset for monitor argument */
 247
 248                 s1 = rd->memuse;
 249
 250                 if (opt_verbosecall) {
 251                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 252
 253                         for (p = 0; p < INT_ARG_CNT; p++)
 254                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 255
 256                         for (p = 0; p < FLT_ARG_CNT; p++)
 257                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 258
 259                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 260                 }
 261
 262                 /* decide which monitor enter function to call */
 263
 264                 if (m->flags & ACC_STATIC) {
 265                         M_MOV_IMM(&m->clazz->object.header, REG_A0);
 266                 }
 267                 else {
 268                         M_TEST(REG_A0);
 269                         M_BNE(8);
 270                         M_ALD_MEM(REG_A0, TRAP_NullPointerException);
 271                 }
 272
 273                 M_AST(REG_A0, REG_SP, s1 * 8);
 274                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 275                 M_CALL(REG_ITMP1);
 276
 277                 if (opt_verbosecall) {
 278                         for (p = 0; p < INT_ARG_CNT; p++)
 279                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 280
 281                         for (p = 0; p < FLT_ARG_CNT; p++)
 282                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 283
 284                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 285                 }
 286         }
 287 #endif
 288
 289 #if !defined(NDEBUG)
 290         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 291                 emit_verbosecall_enter(jd);
 292 #endif /* !defined(NDEBUG) */
 293
 294         }
 295
 296         /* end of header generation */
 297
 298         /* create replacement points */
 299
 300         REPLACEMENT_POINTS_INIT(cd, jd);
 301
 302         /* walk through all basic blocks */
 303
 304         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 305
 306                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 307
 308                 if (bptr->flags >= BBREACHED) {
 309
 310                 /* branch resolving */
 311
 312                 codegen_resolve_branchrefs(cd, bptr);
 313
 314                 /* handle replacement points */
 315
 316                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 317
 318                 /* copy interface registers to their destination */
 319
 320                 len = bptr->indepth;
 321                 MCODECHECK(512);
 322
 323 #if defined(ENABLE_PROFILING)
 324                 /* generate basicblock profiling code */
 325
 326                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 327                         /* count frequency */
 328
 329                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 330                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 331
 332                         /* if this is an exception handler, start profiling again */
 333
 334                         if (bptr->type == BBTYPE_EXH)
 335                                 PROFILE_CYCLE_START;
 336                 }
 337 #endif
 338
 339 #if defined(ENABLE_LSRA)
 340                 if (opt_lsra) {
 341                         while (len) {
 342                                 len--;
 343                                 src = bptr->invars[len];
 344                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 345                                         if (bptr->type == BBTYPE_EXH) {
 346 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 347                                                 if (!IS_INMEMORY(src->flags))
 348                                                         d= src->vv.regoff;
 349                                                 else
 350                                                         d=REG_ITMP1;
 351                                                 M_INTMOVE(REG_ITMP1, d);
 352                                                 emit_store(jd, NULL, src, d);
 353                                         }
 354                                 }
 355                         }
 356
 357                 } else {
 358 #endif
 359
 360                 while (len) {
 361                         len--;
 362                         var = VAR(bptr->invars[len]);
 363                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 364                                 if (bptr->type == BBTYPE_EXH) {
 365                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 366                                         M_INTMOVE(REG_ITMP1, d);
 367                                         emit_store(jd, NULL, var, d);
 368                                 }
 369                         }
 370                         else {
 371                                 assert((var->flags & INOUT));
 372                         }
 373                 }
 374 #if defined(ENABLE_LSRA)
 375                 }
 376 #endif
 377                 /* walk through all instructions */
 378
 379                 len = bptr->icount;
 380                 currentline = 0;
 381
 382                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 383                         if (iptr->line != currentline) {
 384                                 linenumbertable_list_entry_add(cd, iptr->line);
 385                                 currentline = iptr->line;
 386                         }
 387
 388                         MCODECHECK(1024);                         /* 1KB should be enough */
 389
 390                 switch (iptr->opc) {
 391                 case ICMD_NOP:        /* ...  ==> ...                                 */
 392                 case ICMD_POP:        /* ..., value  ==> ...                          */
 393                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 394                         break;
 395
 396                 case ICMD_INLINE_START:
 397
 398                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 399                         break;
 400
 401                 case ICMD_INLINE_BODY:
 402
 403                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 404                         linenumbertable_list_entry_add_inline_start(cd, iptr);
 405                         linenumbertable_list_entry_add(cd, iptr->line);
 406                         break;
 407
 408                 case ICMD_INLINE_END:
 409
 410                         linenumbertable_list_entry_add_inline_end(cd, iptr);
 411                         linenumbertable_list_entry_add(cd, iptr->line);
 412                         break;
 413
 414                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 415
 416                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 417                         emit_nullpointer_check(cd, iptr, s1);
 418                         break;
 419
 420                 /* constant operations ************************************************/
 421
 422                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 423
 424                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 425                         ICONST(d, iptr->sx.val.i);
 426                         emit_store_dst(jd, iptr, d);
 427                         break;
 428
 429                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 430
 431                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 432                         LCONST(d, iptr->sx.val.l);
 433                         emit_store_dst(jd, iptr, d);
 434                         break;
 435
 436                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 437
 438                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 439                         disp = dseg_add_float(cd, iptr->sx.val.f);
 440                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 441                         emit_store_dst(jd, iptr, d);
 442                         break;
 443
 444                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 445
 446                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 447                         disp = dseg_add_double(cd, iptr->sx.val.d);
 448                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 449                         emit_store_dst(jd, iptr, d);
 450                         break;
 451
 452                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 453
 454                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 455
 456                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 457                                 cr   = iptr->sx.val.c.ref;
 458                                 disp = dseg_add_unique_address(cd, cr);
 459
 460 /*                              PROFILE_CYCLE_STOP; */
 461
 462                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
 463                                                                           cr, disp);
 464
 465 /*                              PROFILE_CYCLE_START; */
 466
 467                                 M_ALD(d, RIP, disp);
 468                         }
 469                         else {
 470                                 if (iptr->sx.val.anyptr == 0) {
 471                                         M_CLR(d);
 472                                 }
 473                                 else {
 474                                         disp = dseg_add_address(cd, iptr->sx.val.anyptr);
 475                                         M_ALD(d, RIP, disp);
 476                                 }
 477                         }
 478                         emit_store_dst(jd, iptr, d);
 479                         break;
 480
 481
 482                 /* load/store/copy/move operations ************************************/
 483
 484                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 485                 case ICMD_ALOAD:      /* s1 = local variable                          */
 486                 case ICMD_LLOAD:
 487                 case ICMD_FLOAD:
 488                 case ICMD_DLOAD:
 489                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 490                 case ICMD_LSTORE:
 491                 case ICMD_FSTORE:
 492                 case ICMD_DSTORE:
 493                 case ICMD_COPY:
 494                 case ICMD_MOVE:
 495
 496                         emit_copy(jd, iptr);
 497                         break;
 498
 499                 case ICMD_ASTORE:
 500                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 501                                 emit_copy(jd, iptr);
 502                         break;
 503
 504                 /* integer operations *************************************************/
 505
 506                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 507
 508                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 509                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 510                         M_INTMOVE(s1, d);
 511                         M_INEG(d);
 512                         emit_store_dst(jd, iptr, d);
 513                         break;
 514
 515                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 516
 517                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 518                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 519                         M_INTMOVE(s1, d);
 520                         M_LNEG(d);
 521                         emit_store_dst(jd, iptr, d);
 522                         break;
 523
 524                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 525
 526                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 527                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 528                         M_ISEXT(s1, d);
 529                         emit_store_dst(jd, iptr, d);
 530                         break;
 531
 532                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 533
 534                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 535                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 536                         M_IMOV(s1, d);
 537                         emit_store_dst(jd, iptr, d);
 538                         break;
 539
 540                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 541
 542                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 543                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 544                         M_BSEXT(s1, d);
 545                         emit_store_dst(jd, iptr, d);
 546                         break;
 547
 548                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 549
 550                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 551                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 552                         M_CZEXT(s1, d);
 553                         emit_store_dst(jd, iptr, d);
 554                         break;
 555
 556                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 557
 558                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 559                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 560                         M_SSEXT(s1, d);
 561                         emit_store_dst(jd, iptr, d);
 562                         break;
 563
 564
 565                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 566
 567                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 568                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 569                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 570                         if (s2 == d)
 571                                 M_IADD(s1, d);
 572                         else {
 573                                 M_INTMOVE(s1, d);
 574                                 M_IADD(s2, d);
 575                         }
 576                         emit_store_dst(jd, iptr, d);
 577                         break;
 578
 579                 case ICMD_IINC:
 580                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 581                                       /* sx.val.i = constant                             */
 582
 583                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 584                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 585
 586                         /* Using inc and dec is not faster than add (tested with
 587                            sieve). */
 588
 589                         M_INTMOVE(s1, d);
 590                         M_IADD_IMM(iptr->sx.val.i, d);
 591                         emit_store_dst(jd, iptr, d);
 592                         break;
 593
 594                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 595
 596                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 597                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 598                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 599                         if (s2 == d)
 600                                 M_LADD(s1, d);
 601                         else {
 602                                 M_INTMOVE(s1, d);
 603                                 M_LADD(s2, d);
 604                         }
 605                         emit_store_dst(jd, iptr, d);
 606                         break;
 607
 608                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 609                                       /* sx.val.l = constant                             */
 610
 611                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 612                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 613                         M_INTMOVE(s1, d);
 614                         if (IS_IMM32(iptr->sx.val.l))
 615                                 M_LADD_IMM(iptr->sx.val.l, d);
 616                         else {
 617                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 618                                 M_LADD(REG_ITMP2, d);
 619                         }
 620                         emit_store_dst(jd, iptr, d);
 621                         break;
 622
 623                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 624
 625                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 626                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 627                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 628                         if (s2 == d) {
 629                                 M_INTMOVE(s1, REG_ITMP1);
 630                                 M_ISUB(s2, REG_ITMP1);
 631                                 M_INTMOVE(REG_ITMP1, d);
 632                         } else {
 633                                 M_INTMOVE(s1, d);
 634                                 M_ISUB(s2, d);
 635                         }
 636                         emit_store_dst(jd, iptr, d);
 637                         break;
 638
 639                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 640                                       /* sx.val.i = constant                             */
 641
 642                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 643                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 644                         M_INTMOVE(s1, d);
 645                         M_ISUB_IMM(iptr->sx.val.i, d);
 646                         emit_store_dst(jd, iptr, d);
 647                         break;
 648
 649                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 650
 651                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 652                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 653                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 654                         if (s2 == d) {
 655                                 M_INTMOVE(s1, REG_ITMP1);
 656                                 M_LSUB(s2, REG_ITMP1);
 657                                 M_INTMOVE(REG_ITMP1, d);
 658                         } else {
 659                                 M_INTMOVE(s1, d);
 660                                 M_LSUB(s2, d);
 661                         }
 662                         emit_store_dst(jd, iptr, d);
 663                         break;
 664
 665                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 666                                       /* sx.val.l = constant                             */
 667
 668                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 669                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 670                         M_INTMOVE(s1, d);
 671                         if (IS_IMM32(iptr->sx.val.l))
 672                                 M_LSUB_IMM(iptr->sx.val.l, d);
 673                         else {
 674                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 675                                 M_LSUB(REG_ITMP2, d);
 676                         }
 677                         emit_store_dst(jd, iptr, d);
 678                         break;
 679
 680                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 681
 682                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 683                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 684                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 685                         if (s2 == d)
 686                                 M_IMUL(s1, d);
 687                         else {
 688                                 M_INTMOVE(s1, d);
 689                                 M_IMUL(s2, d);
 690                         }
 691                         emit_store_dst(jd, iptr, d);
 692                         break;
 693
 694                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 695                                       /* sx.val.i = constant                             */
 696
 697                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 698                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 699                         if (iptr->sx.val.i == 2) {
 700                                 M_INTMOVE(s1, d);
 701                                 M_ISLL_IMM(1, d);
 702                         } else
 703                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 704                         emit_store_dst(jd, iptr, d);
 705                         break;
 706
 707                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 708
 709                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 710                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 711                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 712                         if (s2 == d)
 713                                 M_LMUL(s1, d);
 714                         else {
 715                                 M_INTMOVE(s1, d);
 716                                 M_LMUL(s2, d);
 717                         }
 718                         emit_store_dst(jd, iptr, d);
 719                         break;
 720
 721                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 722                                       /* sx.val.l = constant                             */
 723
 724                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 725                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 726                         if (IS_IMM32(iptr->sx.val.l))
 727                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 728                         else {
 729                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 730                                 M_INTMOVE(s1, d);
 731                                 M_LMUL(REG_ITMP2, d);
 732                         }
 733                         emit_store_dst(jd, iptr, d);
 734                         break;
 735
 736                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 737
 738                         s1 = emit_load_s1(jd, iptr, RAX);
 739                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 740                         d = codegen_reg_of_dst(jd, iptr, RAX);
 741
 742                         M_INTMOVE(s1, RAX);
 743                         M_INTMOVE(s2, REG_ITMP3);
 744                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 745
 746                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 747
 748                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 749                         M_BNE(4 + 6);
 750                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 751                         M_BEQ(1 + 3);                                          /* 6 bytes */
 752
 753                         emit_cltd(cd);                                         /* 1 byte  */
 754                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 755
 756                         M_INTMOVE(RAX, d);
 757                         emit_store_dst(jd, iptr, d);
 758                         dst = VAROP(iptr->dst);
 759                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 760                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 761                         break;
 762
 763                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 764
 765                         s1 = emit_load_s1(jd, iptr, RAX);
 766                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 767                         d = codegen_reg_of_dst(jd, iptr, RDX);
 768
 769                         M_INTMOVE(s1, RAX);
 770                         M_INTMOVE(s2, REG_ITMP3);
 771                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 772
 773                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 774
 775                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 776                         M_BNE(3 + 4 + 6);
 777                         M_CLR(RDX);                                            /* 3 bytes */
 778                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 779                         M_BEQ(1 + 3);                                          /* 6 bytes */
 780
 781                         emit_cltd(cd);                                         /* 1 byte  */
 782                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 783
 784                         M_INTMOVE(RDX, d);
 785                         emit_store_dst(jd, iptr, d);
 786                         dst = VAROP(iptr->dst);
 787                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 788                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 789                         break;
 790
 791                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 792                                       /* sx.val.i = constant                             */
 793
 794                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 795                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 796                         M_INTMOVE(s1, REG_ITMP1);
 797                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 798                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 799                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 800                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 801                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 802                         emit_store_dst(jd, iptr, d);
 803                         break;
 804
 805                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 806                                       /* sx.val.i = constant                             */
 807
 808                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 809                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 810                         M_INTMOVE(s1, REG_ITMP1);
 811                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 812                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 813                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 814                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 815                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 816                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 817                         emit_store_dst(jd, iptr, d);
 818                         break;
 819
 820
 821                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 822
 823                         s1 = emit_load_s1(jd, iptr, RAX);
 824                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 825                         d = codegen_reg_of_dst(jd, iptr, RAX);
 826
 827                         M_INTMOVE(s1, RAX);
 828                         M_INTMOVE(s2, REG_ITMP3);
 829                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 830
 831                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 832
 833                         /* check as described in jvm spec */
 834                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 835                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 836                         M_BNE(4 + 6);
 837                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 838                         M_BEQ(2 + 3);                                          /* 6 bytes */
 839
 840                         emit_cqto(cd);                                         /* 2 bytes */
 841                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 842
 843                         M_INTMOVE(RAX, d);
 844                         emit_store_dst(jd, iptr, d);
 845                         dst = VAROP(iptr->dst);
 846                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 847                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 848                         break;
 849
 850                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 851
 852                         s1 = emit_load_s1(jd, iptr, RAX);
 853                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 854                         d = codegen_reg_of_dst(jd, iptr, RDX);
 855
 856                         M_INTMOVE(s1, RAX);
 857                         M_INTMOVE(s2, REG_ITMP3);
 858                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 859
 860                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 861
 862                         /* check as described in jvm spec */
 863                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 864                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 865                         M_BNE(3 + 4 + 6);
 866                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 867                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 868                         M_BEQ(2 + 3);                                          /* 6 bytes */
 869
 870                         emit_cqto(cd);                                         /* 2 bytes */
 871                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 872
 873                         M_INTMOVE(RDX, d);
 874                         emit_store_dst(jd, iptr, d);
 875                         dst = VAROP(iptr->dst);
 876                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 877                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 878                         break;
 879
 880                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 881                                       /* sx.val.i = constant                             */
 882
 883                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 884                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 885                         M_INTMOVE(s1, REG_ITMP1);
 886                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 887                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 888                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 889                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 890                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 891                         emit_store_dst(jd, iptr, d);
 892                         break;
 893
 894                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 895                                       /* sx.val.l = constant                             */
 896
 897                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 898                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 899                         M_INTMOVE(s1, REG_ITMP1);
 900                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 901                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 902                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 903                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 904                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 905                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 906                         emit_store_dst(jd, iptr, d);
 907                         break;
 908
 909                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 910
 911                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 912                         emit_ishift(jd, SHIFT_SHL, iptr);
 913                         break;
 914
 915                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 916                                       /* sx.val.i = constant                             */
 917
 918                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 919                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 920                         M_INTMOVE(s1, d);
 921                         M_ISLL_IMM(iptr->sx.val.i, d);
 922                         emit_store_dst(jd, iptr, d);
 923                         break;
 924
 925                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 926
 927                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 928                         emit_ishift(jd, SHIFT_SAR, iptr);
 929                         break;
 930
 931                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 932                                       /* sx.val.i = constant                             */
 933
 934                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 935                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 936                         M_INTMOVE(s1, d);
 937                         M_ISRA_IMM(iptr->sx.val.i, d);
 938                         emit_store_dst(jd, iptr, d);
 939                         break;
 940
 941                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 942
 943                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 944                         emit_ishift(jd, SHIFT_SHR, iptr);
 945                         break;
 946
 947                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 948                                       /* sx.val.i = constant                             */
 949
 950                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 951                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 952                         M_INTMOVE(s1, d);
 953                         M_ISRL_IMM(iptr->sx.val.i, d);
 954                         emit_store_dst(jd, iptr, d);
 955                         break;
 956
 957                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 958
 959                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 960                         emit_lshift(jd, SHIFT_SHL, iptr);
 961                         break;
 962
 963         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 964                                           /* sx.val.i = constant                             */
 965
 966                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 967                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 968                         M_INTMOVE(s1, d);
 969                         M_LSLL_IMM(iptr->sx.val.i, d);
 970                         emit_store_dst(jd, iptr, d);
 971                         break;
 972
 973                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 974
 975                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 976                         emit_lshift(jd, SHIFT_SAR, iptr);
 977                         break;
 978
 979                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 980                                       /* sx.val.i = constant                             */
 981
 982                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 983                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 984                         M_INTMOVE(s1, d);
 985                         M_LSRA_IMM(iptr->sx.val.i, d);
 986                         emit_store_dst(jd, iptr, d);
 987                         break;
 988
 989                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 990
 991                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 992                         emit_lshift(jd, SHIFT_SHR, iptr);
 993                         break;
 994
 995                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 996                                       /* sx.val.l = constant                             */
 997
 998                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 999                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1000                         M_INTMOVE(s1, d);
1001                         M_LSRL_IMM(iptr->sx.val.i, d);
1002                         emit_store_dst(jd, iptr, d);
1003                         break;
1004
1005                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1006
1007                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1008                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1009                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1010                         if (s2 == d)
1011                                 M_IAND(s1, d);
1012                         else {
1013                                 M_INTMOVE(s1, d);
1014                                 M_IAND(s2, d);
1015                         }
1016                         emit_store_dst(jd, iptr, d);
1017                         break;
1018
1019                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1020                                       /* sx.val.i = constant                             */
1021
1022                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1023                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1024                         M_INTMOVE(s1, d);
1025                         M_IAND_IMM(iptr->sx.val.i, d);
1026                         emit_store_dst(jd, iptr, d);
1027                         break;
1028
1029                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1030
1031                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1032                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1033                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1034                         if (s2 == d)
1035                                 M_LAND(s1, d);
1036                         else {
1037                                 M_INTMOVE(s1, d);
1038                                 M_LAND(s2, d);
1039                         }
1040                         emit_store_dst(jd, iptr, d);
1041                         break;
1042
1043                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1044                                       /* sx.val.l = constant                             */
1045
1046                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1047                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1048                         M_INTMOVE(s1, d);
1049                         if (IS_IMM32(iptr->sx.val.l))
1050                                 M_LAND_IMM(iptr->sx.val.l, d);
1051                         else {
1052                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1053                                 M_LAND(REG_ITMP2, d);
1054                         }
1055                         emit_store_dst(jd, iptr, d);
1056                         break;
1057
1058                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1059
1060                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1061                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1062                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1063                         if (s2 == d)
1064                                 M_IOR(s1, d);
1065                         else {
1066                                 M_INTMOVE(s1, d);
1067                                 M_IOR(s2, d);
1068                         }
1069                         emit_store_dst(jd, iptr, d);
1070                         break;
1071
1072                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1073                                       /* sx.val.i = constant                             */
1074
1075                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1076                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1077                         M_INTMOVE(s1, d);
1078                         M_IOR_IMM(iptr->sx.val.i, d);
1079                         emit_store_dst(jd, iptr, d);
1080                         break;
1081
1082                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1083
1084                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1085                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1086                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1087                         if (s2 == d)
1088                                 M_LOR(s1, d);
1089                         else {
1090                                 M_INTMOVE(s1, d);
1091                                 M_LOR(s2, d);
1092                         }
1093                         emit_store_dst(jd, iptr, d);
1094                         break;
1095
1096                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1097                                       /* sx.val.l = constant                             */
1098
1099                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1100                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1101                         M_INTMOVE(s1, d);
1102                         if (IS_IMM32(iptr->sx.val.l))
1103                                 M_LOR_IMM(iptr->sx.val.l, d);
1104                         else {
1105                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1106                                 M_LOR(REG_ITMP2, d);
1107                         }
1108                         emit_store_dst(jd, iptr, d);
1109                         break;
1110
1111                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1112
1113                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1114                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1115                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1116                         if (s2 == d)
1117                                 M_IXOR(s1, d);
1118                         else {
1119                                 M_INTMOVE(s1, d);
1120                                 M_IXOR(s2, d);
1121                         }
1122                         emit_store_dst(jd, iptr, d);
1123                         break;
1124
1125                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1126                                       /* sx.val.i = constant                             */
1127
1128                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1129                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1130                         M_INTMOVE(s1, d);
1131                         M_IXOR_IMM(iptr->sx.val.i, d);
1132                         emit_store_dst(jd, iptr, d);
1133                         break;
1134
1135                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1136
1137                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1138                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1139                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1140                         if (s2 == d)
1141                                 M_LXOR(s1, d);
1142                         else {
1143                                 M_INTMOVE(s1, d);
1144                                 M_LXOR(s2, d);
1145                         }
1146                         emit_store_dst(jd, iptr, d);
1147                         break;
1148
1149                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1150                                       /* sx.val.l = constant                             */
1151
1152                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1153                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1154                         M_INTMOVE(s1, d);
1155                         if (IS_IMM32(iptr->sx.val.l))
1156                                 M_LXOR_IMM(iptr->sx.val.l, d);
1157                         else {
1158                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1159                                 M_LXOR(REG_ITMP2, d);
1160                         }
1161                         emit_store_dst(jd, iptr, d);
1162                         break;
1163
1164
1165                 /* floating operations ************************************************/
1166
1167                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1168
1169                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1170                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1171                         disp = dseg_add_s4(cd, 0x80000000);
1172                         M_FLTMOVE(s1, d);
1173                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1174                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1175                         emit_store_dst(jd, iptr, d);
1176                         break;
1177
1178                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1179
1180                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1181                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1182                         disp = dseg_add_s8(cd, 0x8000000000000000);
1183                         M_FLTMOVE(s1, d);
1184                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1185                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1186                         emit_store_dst(jd, iptr, d);
1187                         break;
1188
1189                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1190
1191                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1192                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1193                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1194                         if (s2 == d)
1195                                 M_FADD(s1, d);
1196                         else {
1197                                 M_FLTMOVE(s1, d);
1198                                 M_FADD(s2, d);
1199                         }
1200                         emit_store_dst(jd, iptr, d);
1201                         break;
1202
1203                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1204
1205                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1206                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1207                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1208                         if (s2 == d)
1209                                 M_DADD(s1, d);
1210                         else {
1211                                 M_FLTMOVE(s1, d);
1212                                 M_DADD(s2, d);
1213                         }
1214                         emit_store_dst(jd, iptr, d);
1215                         break;
1216
1217                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1218
1219                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1220                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1221                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1222                         if (s2 == d) {
1223                                 M_FLTMOVE(s2, REG_FTMP2);
1224                                 s2 = REG_FTMP2;
1225                         }
1226                         M_FLTMOVE(s1, d);
1227                         M_FSUB(s2, d);
1228                         emit_store_dst(jd, iptr, d);
1229                         break;
1230
1231                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1232
1233                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1234                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1235                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1236                         if (s2 == d) {
1237                                 M_FLTMOVE(s2, REG_FTMP2);
1238                                 s2 = REG_FTMP2;
1239                         }
1240                         M_FLTMOVE(s1, d);
1241                         M_DSUB(s2, d);
1242                         emit_store_dst(jd, iptr, d);
1243                         break;
1244
1245                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1246
1247                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1248                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1249                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1250                         if (s2 == d)
1251                                 M_FMUL(s1, d);
1252                         else {
1253                                 M_FLTMOVE(s1, d);
1254                                 M_FMUL(s2, d);
1255                         }
1256                         emit_store_dst(jd, iptr, d);
1257                         break;
1258
1259                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1260
1261                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1262                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1263                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1264                         if (s2 == d)
1265                                 M_DMUL(s1, d);
1266                         else {
1267                                 M_FLTMOVE(s1, d);
1268                                 M_DMUL(s2, d);
1269                         }
1270                         emit_store_dst(jd, iptr, d);
1271                         break;
1272
1273                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1274
1275                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1276                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1277                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1278                         if (s2 == d) {
1279                                 M_FLTMOVE(s2, REG_FTMP2);
1280                                 s2 = REG_FTMP2;
1281                         }
1282                         M_FLTMOVE(s1, d);
1283                         M_FDIV(s2, d);
1284                         emit_store_dst(jd, iptr, d);
1285                         break;
1286
1287                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1288
1289                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1290                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1291                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1292                         if (s2 == d) {
1293                                 M_FLTMOVE(s2, REG_FTMP2);
1294                                 s2 = REG_FTMP2;
1295                         }
1296                         M_FLTMOVE(s1, d);
1297                         M_DDIV(s2, d);
1298                         emit_store_dst(jd, iptr, d);
1299                         break;
1300
1301                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1302
1303                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1304                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1305                         M_CVTIF(s1, d);
1306                         emit_store_dst(jd, iptr, d);
1307                         break;
1308
1309                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1310
1311                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1312                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1313                         M_CVTID(s1, d);
1314                         emit_store_dst(jd, iptr, d);
1315                         break;
1316
1317                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1318
1319                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1320                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1321                         M_CVTLF(s1, d);
1322                         emit_store_dst(jd, iptr, d);
1323                         break;
1324
1325                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1326
1327                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1328                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1329                         M_CVTLD(s1, d);
1330                         emit_store_dst(jd, iptr, d);
1331                         break;
1332
1333                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1334
1335                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1336                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1337                         M_CVTFI(s1, d);
1338                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1339                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1340                                 ((REG_RESULT == d) ? 0 : 3);
1341                         M_BNE(disp);
1342                         M_FLTMOVE(s1, REG_FTMP1);
1343                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1344                         M_CALL(REG_ITMP2);
1345                         M_INTMOVE(REG_RESULT, d);
1346                         emit_store_dst(jd, iptr, d);
1347                         break;
1348
1349                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1350
1351                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1352                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1353                         M_CVTDI(s1, d);
1354                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1355                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1356                                 ((REG_RESULT == d) ? 0 : 3);
1357                         M_BNE(disp);
1358                         M_FLTMOVE(s1, REG_FTMP1);
1359                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1360                         M_CALL(REG_ITMP2);
1361                         M_INTMOVE(REG_RESULT, d);
1362                         emit_store_dst(jd, iptr, d);
1363                         break;
1364
1365                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1366
1367                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1368                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1369                         M_CVTFL(s1, d);
1370                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1371                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1372                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1373                                 ((REG_RESULT == d) ? 0 : 3);
1374                         M_BNE(disp);
1375                         M_FLTMOVE(s1, REG_FTMP1);
1376                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1377                         M_CALL(REG_ITMP2);
1378                         M_INTMOVE(REG_RESULT, d);
1379                         emit_store_dst(jd, iptr, d);
1380                         break;
1381
1382                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1383
1384                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1385                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1386                         M_CVTDL(s1, d);
1387                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1388                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1389                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1390                                 ((REG_RESULT == d) ? 0 : 3);
1391                         M_BNE(disp);
1392                         M_FLTMOVE(s1, REG_FTMP1);
1393                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1394                         M_CALL(REG_ITMP2);
1395                         M_INTMOVE(REG_RESULT, d);
1396                         emit_store_dst(jd, iptr, d);
1397                         break;
1398
1399                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1400
1401                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1402                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1403                         M_CVTFD(s1, d);
1404                         emit_store_dst(jd, iptr, d);
1405                         break;
1406
1407                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1408
1409                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1410                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1411                         M_CVTDF(s1, d);
1412                         emit_store_dst(jd, iptr, d);
1413                         break;
1414
1415                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1416                                           /* == => 0, < => 1, > => -1 */
1417
1418                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1419                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1420                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1421                         M_CLR(d);
1422                         M_MOV_IMM(1, REG_ITMP1);
1423                         M_MOV_IMM(-1, REG_ITMP2);
1424                         emit_ucomiss_reg_reg(cd, s1, s2);
1425                         M_CMOVULT(REG_ITMP1, d);
1426                         M_CMOVUGT(REG_ITMP2, d);
1427                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1428                         emit_store_dst(jd, iptr, d);
1429                         break;
1430
1431                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1432                                           /* == => 0, < => 1, > => -1 */
1433
1434                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1435                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1436                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1437                         M_CLR(d);
1438                         M_MOV_IMM(1, REG_ITMP1);
1439                         M_MOV_IMM(-1, REG_ITMP2);
1440                         emit_ucomiss_reg_reg(cd, s1, s2);
1441                         M_CMOVULT(REG_ITMP1, d);
1442                         M_CMOVUGT(REG_ITMP2, d);
1443                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1444                         emit_store_dst(jd, iptr, d);
1445                         break;
1446
1447                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1448                                           /* == => 0, < => 1, > => -1 */
1449
1450                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1451                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1452                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1453                         M_CLR(d);
1454                         M_MOV_IMM(1, REG_ITMP1);
1455                         M_MOV_IMM(-1, REG_ITMP2);
1456                         emit_ucomisd_reg_reg(cd, s1, s2);
1457                         M_CMOVULT(REG_ITMP1, d);
1458                         M_CMOVUGT(REG_ITMP2, d);
1459                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1460                         emit_store_dst(jd, iptr, d);
1461                         break;
1462
1463                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1464                                           /* == => 0, < => 1, > => -1 */
1465
1466                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1467                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1468                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1469                         M_CLR(d);
1470                         M_MOV_IMM(1, REG_ITMP1);
1471                         M_MOV_IMM(-1, REG_ITMP2);
1472                         emit_ucomisd_reg_reg(cd, s1, s2);
1473                         M_CMOVULT(REG_ITMP1, d);
1474                         M_CMOVUGT(REG_ITMP2, d);
1475                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1476                         emit_store_dst(jd, iptr, d);
1477                         break;
1478
1479
1480                 /* memory operations **************************************************/
1481
1482                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1483
1484                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1485                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1486                         /* implicit null-pointer check */
1487                         M_ILD(d, s1, OFFSET(java_array_t, size));
1488                         emit_store_dst(jd, iptr, d);
1489                         break;
1490
1491                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1492
1493                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1494                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1495                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1496                         /* implicit null-pointer check */
1497                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1498                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1499                         emit_store_dst(jd, iptr, d);
1500                         break;
1501
1502                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1503
1504                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1505                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1506                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1507                         /* implicit null-pointer check */
1508                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1509                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1510                         emit_store_dst(jd, iptr, d);
1511                         break;
1512
1513                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1514
1515                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1516                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1517                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1518                         /* implicit null-pointer check */
1519                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1520                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1521                         emit_store_dst(jd, iptr, d);
1522                         break;
1523
1524                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1525
1526                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1527                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1528                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1529                         /* implicit null-pointer check */
1530                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1531                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1532                         emit_store_dst(jd, iptr, d);
1533                         break;
1534
1535                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1536
1537                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1538                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1539                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1540                         /* implicit null-pointer check */
1541                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1542                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1543                         emit_store_dst(jd, iptr, d);
1544                         break;
1545
1546                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1547
1548                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1549                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1550                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1551                         /* implicit null-pointer check */
1552                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1553                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1554                         emit_store_dst(jd, iptr, d);
1555                         break;
1556
1557                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1558
1559                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1560                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1561                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1562                         /* implicit null-pointer check */
1563                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1564                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1565                         emit_store_dst(jd, iptr, d);
1566                         break;
1567
1568                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1569
1570                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1571                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1572                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1573                         /* implicit null-pointer check */
1574                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1575                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1576                         emit_store_dst(jd, iptr, d);
1577                         break;
1578
1579
1580                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1581
1582                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1583                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1584                         /* implicit null-pointer check */
1585                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1586                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1587                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1588                         break;
1589
1590                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1591
1592                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1593                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1594                         /* implicit null-pointer check */
1595                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1596                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1597                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1598                         break;
1599
1600                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1601
1602                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1603                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1604                         /* implicit null-pointer check */
1605                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1606                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1607                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1608                         break;
1609
1610                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1611
1612                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1613                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1614                         /* implicit null-pointer check */
1615                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1616                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1617                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1618                         break;
1619
1620                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1621
1622                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1623                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1624                         /* implicit null-pointer check */
1625                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1626                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1627                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1628                         break;
1629
1630                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1631
1632                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1633                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1634                         /* implicit null-pointer check */
1635                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1636                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1637                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1638                         break;
1639
1640                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1641
1642                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1643                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1644                         /* implicit null-pointer check */
1645                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1646                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1647                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1648                         break;
1649
1650                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1651
1652                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1653                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1654                         /* implicit null-pointer check */
1655                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1656                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1657
1658                         M_MOV(s1, REG_A0);
1659                         M_MOV(s3, REG_A1);
1660                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
1661                         M_CALL(REG_ITMP1);
1662                         emit_arraystore_check(cd, iptr);
1663
1664                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1665                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1666                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1667                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1668                         break;
1669
1670
1671                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1672
1673                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1674                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1675                         /* implicit null-pointer check */
1676                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1677                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1678                         break;
1679
1680                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1681
1682                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1683                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1684                         /* implicit null-pointer check */
1685                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1686                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1687                         break;
1688
1689                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1690
1691                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1692                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1693                         /* implicit null-pointer check */
1694                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1695                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1696                         break;
1697
1698                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1699
1700                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1701                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1702                         /* implicit null-pointer check */
1703                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1704                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1705                         break;
1706
1707                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1708
1709                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1710                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1711                         /* implicit null-pointer check */
1712                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1713
1714                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1715                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1716                         }
1717                         else {
1718                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1719                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1720                         }
1721                         break;
1722
1723                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1724
1725                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1726                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1727                         /* implicit null-pointer check */
1728                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1729                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1730                         break;
1731
1732
1733                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1734
1735                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1736                                 uf        = iptr->sx.s23.s3.uf;
1737                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1738                                 disp      = dseg_add_unique_address(cd, uf);
1739
1740 /*                              PROFILE_CYCLE_STOP; */
1741
1742                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1743
1744 /*                              PROFILE_CYCLE_START; */
1745                         }
1746                         else {
1747                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1748                                 fieldtype = fi->type;
1749                                 disp      = dseg_add_address(cd, fi->value);
1750
1751                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1752                                         PROFILE_CYCLE_STOP;
1753
1754                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1755                                                                                   fi->clazz, 0);
1756
1757                                         PROFILE_CYCLE_START;
1758                                 }
1759                         }
1760
1761                         /* This approach is much faster than moving the field
1762                            address inline into a register. */
1763
1764                         M_ALD(REG_ITMP1, RIP, disp);
1765
1766                         switch (fieldtype) {
1767                         case TYPE_INT:
1768                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1769                                 M_ILD(d, REG_ITMP1, 0);
1770                                 break;
1771                         case TYPE_LNG:
1772                         case TYPE_ADR:
1773                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1774                                 M_LLD(d, REG_ITMP1, 0);
1775                                 break;
1776                         case TYPE_FLT:
1777                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1778                                 M_FLD(d, REG_ITMP1, 0);
1779                                 break;
1780                         case TYPE_DBL:
1781                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1782                                 M_DLD(d, REG_ITMP1, 0);
1783                                 break;
1784                         }
1785                         emit_store_dst(jd, iptr, d);
1786                         break;
1787
1788                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1789
1790                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1791                                 uf        = iptr->sx.s23.s3.uf;
1792                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1793                                 disp      = dseg_add_unique_address(cd, uf);
1794
1795 /*                              PROFILE_CYCLE_STOP; */
1796
1797                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1798
1799 /*                              PROFILE_CYCLE_START; */
1800                         }
1801                         else {
1802                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1803                                 fieldtype = fi->type;
1804                                 disp      = dseg_add_address(cd, fi->value);
1805
1806                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1807                                         PROFILE_CYCLE_STOP;
1808
1809                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1810                                                                                   fi->clazz, 0);
1811
1812                                         PROFILE_CYCLE_START;
1813                                 }
1814                         }
1815
1816                         /* This approach is much faster than moving the field
1817                            address inline into a register. */
1818
1819                         M_ALD(REG_ITMP1, RIP, disp);
1820
1821                         switch (fieldtype) {
1822                         case TYPE_INT:
1823                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1824                                 M_IST(s1, REG_ITMP1, 0);
1825                                 break;
1826                         case TYPE_LNG:
1827                         case TYPE_ADR:
1828                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1829                                 M_LST(s1, REG_ITMP1, 0);
1830                                 break;
1831                         case TYPE_FLT:
1832                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1833                                 M_FST(s1, REG_ITMP1, 0);
1834                                 break;
1835                         case TYPE_DBL:
1836                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1837                                 M_DST(s1, REG_ITMP1, 0);
1838                                 break;
1839                         }
1840                         break;
1841
1842                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1843                                           /* val = value (in current instruction)     */
1844                                           /* following NOP)                           */
1845
1846                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1847                                 uf        = iptr->sx.s23.s3.uf;
1848                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1849                                 disp      = dseg_add_unique_address(cd, uf);
1850
1851 /*                              PROFILE_CYCLE_STOP; */
1852
1853                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1854
1855 /*                              PROFILE_CYCLE_START; */
1856                         }
1857                         else {
1858                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1859                                 fieldtype = fi->type;
1860                                 disp      = dseg_add_address(cd, fi->value);
1861
1862                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1863                                         PROFILE_CYCLE_STOP;
1864
1865                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1866                                                                                   fi->clazz, 0);
1867
1868                                         PROFILE_CYCLE_START;
1869                                 }
1870                         }
1871
1872                         /* This approach is much faster than moving the field
1873                            address inline into a register. */
1874
1875                         M_ALD(REG_ITMP1, RIP, disp);
1876
1877                         switch (fieldtype) {
1878                         case TYPE_INT:
1879                         case TYPE_FLT:
1880                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1881                                 break;
1882                         case TYPE_LNG:
1883                         case TYPE_ADR:
1884                         case TYPE_DBL:
1885                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1886                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1887                                 else {
1888                                         M_MOV_IMM(iptr->sx.s23.s2.constval, REG_ITMP2);
1889                                         M_LST(REG_ITMP2, REG_ITMP1, 0);
1890                                 }
1891                                 break;
1892                         }
1893                         break;
1894
1895                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1896
1897                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1898
1899                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1900                                 uf        = iptr->sx.s23.s3.uf;
1901                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1902                                 disp      = 0;
1903
1904 /*                              PROFILE_CYCLE_STOP; */
1905
1906                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1907
1908 /*                              PROFILE_CYCLE_START; */
1909                         }
1910                         else {
1911                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1912                                 fieldtype = fi->type;
1913                                 disp      = fi->offset;
1914                         }
1915
1916                         /* implicit null-pointer check */
1917                         switch (fieldtype) {
1918                         case TYPE_INT:
1919                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1920                                 M_ILD32(d, s1, disp);
1921                                 break;
1922                         case TYPE_LNG:
1923                         case TYPE_ADR:
1924                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1925                                 M_LLD32(d, s1, disp);
1926                                 break;
1927                         case TYPE_FLT:
1928                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1929                                 M_FLD32(d, s1, disp);
1930                                 break;
1931                         case TYPE_DBL:
1932                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1933                                 M_DLD32(d, s1, disp);
1934                                 break;
1935                         }
1936                         emit_store_dst(jd, iptr, d);
1937                         break;
1938
1939                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1940
1941                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1942                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1943
1944                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1945                                 uf        = iptr->sx.s23.s3.uf;
1946                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1947                                 disp      = 0;
1948
1949 /*                              PROFILE_CYCLE_STOP; */
1950
1951                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1952
1953 /*                              PROFILE_CYCLE_START; */
1954                         }
1955                         else {
1956                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1957                                 fieldtype = fi->type;
1958                                 disp      = fi->offset;
1959                         }
1960
1961                         /* implicit null-pointer check */
1962                         switch (fieldtype) {
1963                         case TYPE_INT:
1964                                 M_IST32(s2, s1, disp);
1965                                 break;
1966                         case TYPE_LNG:
1967                         case TYPE_ADR:
1968                                 M_LST32(s2, s1, disp);
1969                                 break;
1970                         case TYPE_FLT:
1971                                 M_FST32(s2, s1, disp);
1972                                 break;
1973                         case TYPE_DBL:
1974                                 M_DST32(s2, s1, disp);
1975                                 break;
1976                         }
1977                         break;
1978
1979                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
1980                                           /* val = value (in current instruction)     */
1981                                           /* following NOP)                           */
1982
1983                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1984
1985                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1986                                 uf        = iptr->sx.s23.s3.uf;
1987                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1988                                 disp      = 0;
1989
1990 /*                              PROFILE_CYCLE_STOP; */
1991
1992                                 patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
1993
1994 /*                              PROFILE_CYCLE_START; */
1995                         }
1996                         else {
1997                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1998                                 fieldtype = fi->type;
1999                                 disp      = fi->offset;
2000                         }
2001
2002                         /* implicit null-pointer check */
2003                         switch (fieldtype) {
2004                         case TYPE_INT:
2005                         case TYPE_FLT:
2006                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2007                                 break;
2008                         case TYPE_LNG:
2009                         case TYPE_ADR:
2010                         case TYPE_DBL:
2011                                 /* XXX why no check for IS_IMM32? -- probably because of the patcher */
2012                                 M_MOV_IMM(iptr->sx.s23.s2.constval, REG_ITMP2);
2013                                 if (disp)  /* resolved, disp can never be 0 */
2014                                         M_LST(REG_ITMP2, s1, disp);
2015                                 else       /* unresolved */
2016                                         M_LST32(REG_ITMP2, s1, disp);
2017                                 break;
2018                         }
2019                         break;
2020
2021
2022                 /* branch operations **************************************************/
2023
2024                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2025
2026                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2027                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2028
2029                         PROFILE_CYCLE_STOP;
2030
2031 #ifdef ENABLE_VERIFIER
2032                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2033                                 uc = iptr->sx.s23.s2.uc;
2034
2035                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2036                         }
2037 #endif /* ENABLE_VERIFIER */
2038
2039                         M_CALL_IMM(0);                            /* passing exception pc */
2040                         M_POP(REG_ITMP2_XPC);
2041
2042                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2043                         M_JMP(REG_ITMP3);
2044                         break;
2045
2046                 case ICMD_GOTO:         /* ... ==> ...                                */
2047                 case ICMD_RET:
2048
2049                         emit_br(cd, iptr->dst.block);
2050                         ALIGNCODENOP;
2051                         break;
2052
2053                 case ICMD_JSR:          /* ... ==> ...                                */
2054
2055                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2056                         ALIGNCODENOP;
2057                         break;
2058
2059                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2060                 case ICMD_IFNONNULL:
2061
2062                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2063                         M_TEST(s1);
2064                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2065                         break;
2066
2067                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2068                 case ICMD_IFLT:
2069                 case ICMD_IFLE:
2070                 case ICMD_IFNE:
2071                 case ICMD_IFGT:
2072                 case ICMD_IFGE:
2073
2074                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2075                         M_ICMP_IMM(iptr->sx.val.i, s1);
2076                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2077                         break;
2078
2079                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2080                 case ICMD_IF_LNE:
2081                 case ICMD_IF_LLT:
2082                 case ICMD_IF_LGE:
2083                 case ICMD_IF_LGT:
2084                 case ICMD_IF_LLE:
2085
2086                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2087                         if (IS_IMM32(iptr->sx.val.l))
2088                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2089                         else {
2090                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2091                                 M_LCMP(REG_ITMP2, s1);
2092                         }
2093                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2094                         break;
2095
2096                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2097                 case ICMD_IF_ICMPNE:
2098                 case ICMD_IF_ICMPLT:
2099                 case ICMD_IF_ICMPGE:
2100                 case ICMD_IF_ICMPGT:
2101                 case ICMD_IF_ICMPLE:
2102
2103                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2104                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2105                         M_ICMP(s2, s1);
2106                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2107                         break;
2108
2109                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2110                 case ICMD_IF_ACMPNE:
2111
2112                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2113                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2114                         M_LCMP(s2, s1);
2115                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2116                         break;
2117
2118                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2119                 case ICMD_IF_LCMPNE:
2120                 case ICMD_IF_LCMPLT:
2121                 case ICMD_IF_LCMPGE:
2122                 case ICMD_IF_LCMPGT:
2123                 case ICMD_IF_LCMPLE:
2124
2125                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2126                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2127                         M_LCMP(s2, s1);
2128                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2129                         break;
2130
2131                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2132                 case ICMD_LRETURN:
2133
2134                         REPLACEMENT_POINT_RETURN(cd, iptr);
2135                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2136                         M_INTMOVE(s1, REG_RESULT);
2137                         goto nowperformreturn;
2138
2139                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2140
2141                         REPLACEMENT_POINT_RETURN(cd, iptr);
2142                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2143                         M_INTMOVE(s1, REG_RESULT);
2144
2145 #ifdef ENABLE_VERIFIER
2146                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2147                                 uc = iptr->sx.s23.s2.uc;
2148
2149                                 PROFILE_CYCLE_STOP;
2150
2151                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2152
2153                                 PROFILE_CYCLE_START;
2154                         }
2155 #endif /* ENABLE_VERIFIER */
2156                         goto nowperformreturn;
2157
2158                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2159                 case ICMD_DRETURN:
2160
2161                         REPLACEMENT_POINT_RETURN(cd, iptr);
2162                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2163                         M_FLTMOVE(s1, REG_FRESULT);
2164                         goto nowperformreturn;
2165
2166                 case ICMD_RETURN:      /* ...  ==> ...                                */
2167
2168                         REPLACEMENT_POINT_RETURN(cd, iptr);
2169
2170 nowperformreturn:
2171                         {
2172                         s4 i, p;
2173
2174                         p = cd->stackframesize;
2175
2176 #if !defined(NDEBUG)
2177                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2178                                 emit_verbosecall_exit(jd);
2179 #endif /* !defined(NDEBUG) */
2180
2181 #if defined(ENABLE_THREADS)
2182                         if (checksync && code_is_synchronized(code)) {
2183                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2184
2185                                 /* we need to save the proper return value */
2186                                 switch (iptr->opc) {
2187                                 case ICMD_IRETURN:
2188                                 case ICMD_ARETURN:
2189                                 case ICMD_LRETURN:
2190                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2191                                         break;
2192                                 case ICMD_FRETURN:
2193                                 case ICMD_DRETURN:
2194                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2195                                         break;
2196                                 }
2197
2198                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2199                                 M_CALL(REG_ITMP1);
2200
2201                                 /* and now restore the proper return value */
2202                                 switch (iptr->opc) {
2203                                 case ICMD_IRETURN:
2204                                 case ICMD_ARETURN:
2205                                 case ICMD_LRETURN:
2206                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2207                                         break;
2208                                 case ICMD_FRETURN:
2209                                 case ICMD_DRETURN:
2210                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2211                                         break;
2212                                 }
2213                         }
2214 #endif
2215
2216                         /* restore saved registers */
2217
2218                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2219                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2220                         }
2221                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2222                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2223                         }
2224
2225                         /* deallocate stack */
2226
2227                         if (cd->stackframesize)
2228                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2229
2230                         /* generate method profiling code */
2231
2232                         PROFILE_CYCLE_STOP;
2233
2234                         M_RET;
2235                         }
2236                         break;
2237
2238
2239                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2240                         {
2241                                 s4 i, l;
2242                                 branch_target_t *table;
2243
2244                                 table = iptr->dst.table;
2245
2246                                 l = iptr->sx.s23.s2.tablelow;
2247                                 i = iptr->sx.s23.s3.tablehigh;
2248
2249                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2250                                 M_INTMOVE(s1, REG_ITMP1);
2251
2252                                 if (l != 0)
2253                                         M_ISUB_IMM(l, REG_ITMP1);
2254
2255                                 /* number of targets */
2256                                 i = i - l + 1;
2257
2258                 /* range check */
2259
2260                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2261                                 emit_bugt(cd, table[0].block);
2262
2263                                 /* build jump table top down and use address of lowest entry */
2264
2265                                 table += i;
2266
2267                                 while (--i >= 0) {
2268                                         dseg_add_target(cd, table->block);
2269                                         --table;
2270                                 }
2271
2272                                 /* length of dataseg after last dseg_add_target is used
2273                                    by load */
2274
2275                                 M_MOV_IMM(0, REG_ITMP2);
2276                                 dseg_adddata(cd);
2277                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2278                                 M_JMP(REG_ITMP1);
2279                         }
2280                         break;
2281
2282
2283                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2284                         {
2285                                 s4 i;
2286                                 lookup_target_t *lookup;
2287
2288                                 lookup = iptr->dst.lookup;
2289
2290                                 i = iptr->sx.s23.s2.lookupcount;
2291
2292                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2293                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2294
2295                                 while (--i >= 0) {
2296                                         M_ICMP_IMM(lookup->value, s1);
2297                                         emit_beq(cd, lookup->target.block);
2298                                         lookup++;
2299                                 }
2300
2301                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2302                                 ALIGNCODENOP;
2303                         }
2304                         break;
2305
2306
2307                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2308
2309                         REPLACEMENT_POINT_FORGC_BUILTIN(cd, iptr);
2310
2311                         bte = iptr->sx.s23.s3.bte;
2312                         md  = bte->md;
2313                         goto gen_method;
2314
2315                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2316
2317                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2318                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2319                 case ICMD_INVOKEINTERFACE:
2320
2321                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2322
2323                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2324                                 lm = NULL;
2325                                 um = iptr->sx.s23.s3.um;
2326                                 md = um->methodref->parseddesc.md;
2327                         }
2328                         else {
2329                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2330                                 um = NULL;
2331                                 md = lm->parseddesc;
2332                         }
2333
2334 gen_method:
2335                         s3 = md->paramcount;
2336
2337                         MCODECHECK((20 * s3) + 128);
2338
2339                         /* copy arguments to registers or stack location */
2340
2341                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2342                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2343                                 d   = md->params[s3].regoff;
2344
2345                                 /* already preallocated (ARGVAR)? */
2346
2347                                 if (var->flags & PREALLOC)
2348                                         continue;
2349
2350                                 if (IS_INT_LNG_TYPE(var->type)) {
2351                                         if (!md->params[s3].inmemory) {
2352                                                 s1 = emit_load(jd, iptr, var, d);
2353                                                 M_INTMOVE(s1, d);
2354                                         }
2355                                         else {
2356                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2357                                                 M_LST(s1, REG_SP, d);
2358                                         }
2359                                 }
2360                                 else {
2361                                         if (!md->params[s3].inmemory) {
2362                                                 s1 = emit_load(jd, iptr, var, d);
2363                                                 M_FLTMOVE(s1, d);
2364                                         }
2365                                         else {
2366                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2367
2368                                                 if (IS_2_WORD_TYPE(var->type))
2369                                                         M_DST(s1, REG_SP, d);
2370                                                 else
2371                                                         M_FST(s1, REG_SP, d);
2372                                         }
2373                                 }
2374                         }
2375
2376                         /* generate method profiling code */
2377
2378                         PROFILE_CYCLE_STOP;
2379
2380                         switch (iptr->opc) {
2381                         case ICMD_BUILTIN:
2382                                 if (bte->stub == NULL) {
2383                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2384                                 }
2385                                 else {
2386                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2387                                 }
2388                                 M_CALL(REG_ITMP1);
2389                                 break;
2390
2391                         case ICMD_INVOKESPECIAL:
2392                                 emit_nullpointer_check(cd, iptr, REG_A0);
2393                                 /* fall through */
2394
2395                         case ICMD_INVOKESTATIC:
2396                                 if (lm == NULL) {
2397                                         disp = dseg_add_unique_address(cd, um);
2398
2399                                         patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
2400                                                                                   um, disp);
2401                                 }
2402                                 else {
2403                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2404                                 }
2405
2406                                 M_ALD(REG_ITMP2, RIP, disp);
2407                                 M_CALL(REG_ITMP2);
2408                                 break;
2409
2410                         case ICMD_INVOKEVIRTUAL:
2411                                 if (lm == NULL) {
2412                                         patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
2413
2414                                         s1 = 0;
2415                                 }
2416                                 else {
2417                                         s1 = OFFSET(vftbl_t, table[0]) +
2418                                                 sizeof(methodptr) * lm->vftblindex;
2419                                 }
2420
2421                                 /* implicit null-pointer check */
2422                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2423                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2424                                 M_CALL(REG_ITMP3);
2425                                 break;
2426
2427                         case ICMD_INVOKEINTERFACE:
2428                                 if (lm == NULL) {
2429                                         patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
2430
2431                                         s1 = 0;
2432                                         s2 = 0;
2433                                 }
2434                                 else {
2435                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2436                                                 sizeof(methodptr) * lm->clazz->index;
2437
2438                                         s2 = sizeof(methodptr) * (lm - lm->clazz->methods);
2439                                 }
2440
2441                                 /* implicit null-pointer check */
2442                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2443                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2444                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2445                                 M_CALL(REG_ITMP3);
2446                                 break;
2447                         }
2448
2449                         /* generate method profiling code */
2450
2451                         PROFILE_CYCLE_START;
2452
2453                         /* store size of call code in replacement point */
2454
2455                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2456                         REPLACEMENT_POINT_FORGC_BUILTIN_RETURN(cd, iptr);
2457
2458                         /* store return value */
2459
2460                         switch (md->returntype.type) {
2461                         case TYPE_INT:
2462                         case TYPE_LNG:
2463                         case TYPE_ADR:
2464                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2465                                 M_INTMOVE(REG_RESULT, s1);
2466                                 emit_store_dst(jd, iptr, s1);
2467                                 break;
2468                         case TYPE_FLT:
2469                         case TYPE_DBL:
2470                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2471                                 M_FLTMOVE(REG_FRESULT, s1);
2472                                 emit_store_dst(jd, iptr, s1);
2473                                 break;
2474                         default:
2475                                 /* TYPE_VOID */
2476                                 break;
2477                         }
2478                         break;
2479
2480
2481                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2482
2483                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2484                                 /* object type cast-check */
2485
2486                                 classinfo *super;
2487                                 s4         superindex;
2488
2489                                 s4 looptarget;
2490
2491                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2492                                         super      = NULL;
2493                                         superindex = 0;
2494                                 }
2495                                 else {
2496                                         super      = iptr->sx.s23.s3.c.cls;
2497                                         superindex = super->index;
2498                                 }
2499
2500                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2501
2502                                 /* if class is not resolved, check which code to call */
2503
2504                                 if (super == NULL) {
2505                                         M_TEST(s1);
2506                                         emit_label_beq(cd, BRANCH_LABEL_1);
2507
2508                                         patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2509                                                                                   iptr->sx.s23.s3.c.ref, 0);
2510
2511                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2512                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2513                                         emit_label_beq(cd, BRANCH_LABEL_2);
2514                                 }
2515
2516                                 /* interface checkcast code */
2517
2518                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2519                                         if (super != NULL) {
2520                                                 M_TEST(s1);
2521                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2522                                         }
2523
2524                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2525
2526                                         if (super == NULL) {
2527                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
2528                                                                                           iptr->sx.s23.s3.c.ref,
2529                                                                                           0);
2530                                         }
2531
2532                                         M_ILD32(REG_ITMP3,
2533                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2534                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2535                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2536
2537                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2538                                                         OFFSET(vftbl_t, interfacetable[0]) -
2539                                                         superindex * sizeof(methodptr*));
2540                                         M_TEST(REG_ITMP3);
2541                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2542
2543                                         if (super == NULL)
2544                                                 emit_label_br(cd, BRANCH_LABEL_4);
2545                                         else
2546                                                 emit_label(cd, BRANCH_LABEL_3);
2547                                 }
2548
2549                                 /* class checkcast code */
2550
2551                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2552                                         if (super == NULL) {
2553                                                 emit_label(cd, BRANCH_LABEL_2);
2554
2555                                                 cr   = iptr->sx.s23.s3.c.ref;
2556                                                 disp = dseg_add_unique_address(cd, cr);
2557
2558                                                 patcher_add_patch_ref(jd,
2559                                                                                           PATCHER_resolve_classref_to_vftbl,
2560                                                                                           cr, disp);
2561                                         }
2562                                         else {
2563                                                 M_TEST(s1);
2564                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2565
2566                                                 disp = dseg_add_address(cd, super->vftbl);
2567                                         }
2568
2569                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2570                                         M_ALD(REG_ITMP3, RIP, disp);
2571
2572                                         if (super == NULL || super->vftbl->subtype_depth >= DISPLAY_SIZE) {
2573                                         M_AADD_IMM(-32, REG_SP); /* need some stack */
2574                                         if (s1 == REG_ITMP1)
2575                                                 M_AST(REG_ITMP1, REG_SP, -8 + 32);
2576
2577                                         M_ALD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_offset));
2578
2579                                         *(cd->mcodeptr++) = 0x4d;
2580                                         *(cd->mcodeptr++) = 0x3b;
2581                                         *(cd->mcodeptr++) = 0x1c;
2582                                         *(cd->mcodeptr++) = 0x02;
2583                                         /* cmp (ITMP2, ITMP1, 1), ITMP3 */
2584
2585                                         emit_label_beq(cd, BRANCH_LABEL_6); /* good */
2586
2587                                         M_LCMP_IMM(OFFSET(vftbl_t, subtype_display[DISPLAY_SIZE]), REG_ITMP1);
2588                                         emit_classcast_check(cd, iptr, BRANCH_NE, REG_ITMP3, s1);
2589
2590                                         M_AST(REG_ITMP3, REG_SP, -16 + 32);
2591                                         M_AST_IMM32(0, REG_SP, -24 + 32);
2592                                         M_ALD(REG_ITMP1, REG_ITMP2, OFFSET(vftbl_t, subtype_overflow));
2593                                         looptarget = cd->mcodeptr - cd->mcodebase;
2594
2595                                         M_ALD(REG_ITMP3, REG_SP, -24 + 32);
2596                                         M_ICMP_MEMBASE(REG_ITMP2, OFFSET(vftbl_t, subtype_overflow_length), REG_ITMP3);
2597                                         emit_label_bge(cd, BRANCH_LABEL_9); /* throw */
2598
2599                                         *(cd->mcodeptr++) = 0x4e;
2600                                         *(cd->mcodeptr++) = 0x8b;
2601                                         *(cd->mcodeptr++) = 0x1c;
2602                                         *(cd->mcodeptr++) = 0xd8;
2603                                         /* movq (ITMP1, ITMP3, 8), ITMP3 */
2604
2605                                         M_LCMP_MEMBASE(REG_SP, -16 + 32, REG_ITMP3);
2606                                         emit_label_beq(cd, BRANCH_LABEL_7); /* good, pop */
2607
2608                                         M_LINC_MEMBASE(REG_SP, -24 + 32);
2609                                         M_JMP_IMM2(looptarget - (cd->mcodeptr - cd->mcodebase) - 2); /* 1 byte displacement */
2610
2611                                         emit_label(cd, BRANCH_LABEL_9);
2612                                         M_AADD_IMM(32, REG_SP); /* restore stack frame */
2613                                         M_ALD_MEM(REG_ITMP3, TRAP_ClassCastException);
2614
2615                                         emit_label(cd, BRANCH_LABEL_7);
2616
2617                                         emit_label(cd, BRANCH_LABEL_6);
2618
2619                                         if (s1 == REG_ITMP1)
2620                                                 M_ALD(REG_ITMP1, REG_SP, -8 + 32);
2621                                         M_AADD_IMM(32, REG_SP);
2622                                         }
2623                                         else {
2624                                                 assert(super->vftbl->subtype_offset < 0x80);
2625                                                 *(cd->mcodeptr++) = 0x4d;
2626                                                 *(cd->mcodeptr++) = 0x3b;
2627                                                 *(cd->mcodeptr++) = 0x5a;
2628                                                 *(cd->mcodeptr++) = super->vftbl->subtype_offset;
2629                                                 /* cmp off(ITMP1), ITMP2 */
2630
2631                                                 emit_classcast_check(cd, iptr, BRANCH_NE, REG_ITMP3, s1);
2632                                         }
2633
2634                                         if (super != NULL)
2635                                                 emit_label(cd, BRANCH_LABEL_5);
2636                                 }
2637
2638                                 if (super == NULL) {
2639                                         emit_label(cd, BRANCH_LABEL_1);
2640                                         emit_label(cd, BRANCH_LABEL_4);
2641                                 }
2642
2643                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2644                         }
2645                         else {
2646                                 /* array type cast-check */
2647
2648                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2649                                 M_INTMOVE(s1, REG_A0);
2650
2651                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2652                                         cr   = iptr->sx.s23.s3.c.ref;
2653                                         disp = dseg_add_unique_address(cd, cr);
2654
2655                                         patcher_add_patch_ref(jd,
2656                                                                                   PATCHER_resolve_classref_to_classinfo,
2657                                                                                   cr, disp);
2658                                 }
2659                                 else {
2660                                         disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2661                                 }
2662
2663                                 M_ALD(REG_A1, RIP, disp);
2664                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2665                                 M_CALL(REG_ITMP1);
2666
2667                                 /* s1 may have been destroyed over the function call */
2668                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2669                                 M_TEST(REG_RESULT);
2670                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2671
2672                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2673                         }
2674
2675                         M_INTMOVE(s1, d);
2676                         emit_store_dst(jd, iptr, d);
2677                         break;
2678
2679                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2680
2681                         {
2682                         classinfo *super;
2683                         s4         superindex;
2684
2685                         s4 looptarget;
2686
2687                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2688                                 super      = NULL;
2689                                 superindex = 0;
2690                         }
2691                         else {
2692                                 super      = iptr->sx.s23.s3.c.cls;
2693                                 superindex = super->index;
2694                         }
2695
2696                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2697                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2698
2699                         if (s1 == d) {
2700                                 M_INTMOVE(s1, REG_ITMP1);
2701                                 s1 = REG_ITMP1;
2702                         }
2703
2704                         M_CLR(d);
2705
2706                         /* if class is not resolved, check which code to call */
2707
2708                         if (super == NULL) {
2709                                 M_TEST(s1);
2710                                 emit_label_beq(cd, BRANCH_LABEL_1);
2711
2712                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2713                                                                           iptr->sx.s23.s3.c.ref, 0);
2714
2715                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2716                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2717                                 emit_label_beq(cd, BRANCH_LABEL_2);
2718                         }
2719
2720                         /* interface instanceof code */
2721
2722                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2723                                 if (super != NULL) {
2724                                         M_TEST(s1);
2725                                         emit_label_beq(cd, BRANCH_LABEL_3);
2726                                 }
2727
2728                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2729
2730                                 if (super == NULL) {
2731                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
2732                                                                                   iptr->sx.s23.s3.c.ref, 0);
2733                                 }
2734
2735                                 M_ILD32(REG_ITMP3,
2736                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2737                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2738
2739                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2740
2741                                 M_BLE(a);
2742                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2743                                                 OFFSET(vftbl_t, interfacetable[0]) -
2744                                                 superindex * sizeof(methodptr*));
2745                                 M_TEST(REG_ITMP1);
2746                                 M_SETNE(d);
2747
2748                                 if (super == NULL)
2749                                         emit_label_br(cd, BRANCH_LABEL_4);
2750                                 else
2751                                         emit_label(cd, BRANCH_LABEL_3);
2752                         }
2753
2754                         /* class instanceof code */
2755
2756                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2757                                 if (super == NULL) {
2758                                         emit_label(cd, BRANCH_LABEL_2);
2759
2760                                         cr   = iptr->sx.s23.s3.c.ref;
2761                                         disp = dseg_add_unique_address(cd, cr);
2762
2763                                         patcher_add_patch_ref(jd,
2764                                                                                   PATCHER_resolve_classref_to_vftbl,
2765                                                                                   cr, disp);
2766                                 }
2767                                 else {
2768                                         M_TEST(s1);
2769                                         emit_label_beq(cd, BRANCH_LABEL_5);
2770
2771                                         disp = dseg_add_address(cd, super->vftbl);
2772                                 }
2773
2774                                 M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2775                                 M_ALD(REG_ITMP3, RIP, disp);
2776
2777                                 if (super == NULL || super->vftbl->subtype_depth >= DISPLAY_SIZE) {
2778                                 M_AADD_IMM(-32, REG_SP); /* need some stack */
2779                                 M_ALD(REG_ITMP1, REG_ITMP3, OFFSET(vftbl_t, subtype_offset));
2780
2781                                 *(cd->mcodeptr++) = 0x4d;
2782                                 *(cd->mcodeptr++) = 0x3b;
2783                                 *(cd->mcodeptr++) = 0x1c;
2784                                 *(cd->mcodeptr++) = 0x02;
2785                                 /* cmp (ITMP2, ITMP1, 1), ITMP3 */
2786
2787                                 emit_label_bne(cd, BRANCH_LABEL_6);
2788                                 if (d == REG_ITMP2) {
2789                                         M_SETE(d);
2790                                         M_BSEXT(d, d);
2791                                 } else
2792                                         M_LINC(d);
2793                                 emit_label_br(cd, BRANCH_LABEL_7); /* ende true */
2794
2795                                 emit_label(cd, BRANCH_LABEL_6);
2796
2797                                 M_LCMP_IMM(OFFSET(vftbl_t, subtype_display[DISPLAY_SIZE]), REG_ITMP1);
2798                                 emit_label_bne(cd, BRANCH_LABEL_6); /* ende false */
2799
2800                                 M_AST(REG_ITMP3, REG_SP, -16 + 32);
2801                                 M_AST_IMM32(0, REG_SP, -24 + 32);
2802                                 M_ALD(REG_ITMP1, REG_ITMP2, OFFSET(vftbl_t, subtype_overflow));
2803                                 looptarget = cd->mcodeptr - cd->mcodebase;
2804
2805                                 M_ALD(REG_ITMP3, REG_SP, -24 + 32);
2806                                 M_ICMP_MEMBASE(REG_ITMP2, OFFSET(vftbl_t, subtype_overflow_length), REG_ITMP3);
2807                                 emit_label_bge(cd, BRANCH_LABEL_8); /* ende false */
2808
2809                                 *(cd->mcodeptr++) = 0x4e;
2810                                 *(cd->mcodeptr++) = 0x8b;
2811                                 *(cd->mcodeptr++) = 0x1c;
2812                                 *(cd->mcodeptr++) = 0xd8;
2813                                 /* movq (ITMP1, ITMP3, 8), ITMP3 */
2814
2815                                 M_LCMP_MEMBASE(REG_SP, -16 + 32, REG_ITMP3);
2816                                 emit_label_bne(cd, BRANCH_LABEL_9);
2817                                 if (d == REG_ITMP2) {
2818                                         M_SETE(d);
2819                                         M_BSEXT(d, d);
2820                                 } else
2821                                         M_LINC(d);
2822                                 emit_label_br(cd, BRANCH_LABEL_10); /* ende true */
2823                                 emit_label(cd, BRANCH_LABEL_9);
2824
2825                                 M_LINC_MEMBASE(REG_SP, -24 + 32);
2826                                 M_JMP_IMM2(looptarget - (cd->mcodeptr - cd->mcodebase) - 2); /* 1 byte displacement */
2827
2828                                 emit_label(cd, BRANCH_LABEL_8);
2829                                 emit_label(cd, BRANCH_LABEL_6);
2830
2831                                 if (d == REG_ITMP2)
2832                                         M_CLR(d);
2833
2834                                 emit_label(cd, BRANCH_LABEL_10);
2835                                 emit_label(cd, BRANCH_LABEL_7);
2836                                 M_AADD_IMM(32, REG_SP);
2837                                 }
2838                                 else {
2839                                         assert(super->vftbl->subtype_offset < 0x80);
2840                                         *(cd->mcodeptr++) = 0x4d;
2841                                         *(cd->mcodeptr++) = 0x3b;
2842                                         *(cd->mcodeptr++) = 0x5a;
2843                                         *(cd->mcodeptr++) = super->vftbl->subtype_offset;
2844                                         /* cmp off(ITMP1), ITMP2 */
2845
2846                                         M_SETE(d);
2847                                         if (d == REG_ITMP2)
2848                                                 M_BSEXT(d, d);
2849                                 }
2850
2851                                 if (super != NULL)
2852                                         emit_label(cd, BRANCH_LABEL_5);
2853                         }
2854
2855                         if (super == NULL) {
2856                                 emit_label(cd, BRANCH_LABEL_1);
2857                                 emit_label(cd, BRANCH_LABEL_4);
2858                         }
2859
2860                         emit_store_dst(jd, iptr, d);
2861                         }
2862                         break;
2863
2864                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2865
2866                         /* check for negative sizes and copy sizes to stack if necessary  */
2867
2868                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2869
2870                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2871
2872                                 /* copy SAVEDVAR sizes to stack */
2873                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2874
2875                                 /* Already Preallocated? */
2876                                 if (!(var->flags & PREALLOC)) {
2877                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2878                                         M_LST(s2, REG_SP, s1 * 8);
2879                                 }
2880                         }
2881
2882                         /* a0 = dimension count */
2883
2884                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2885
2886                         /* is a patcher function set? */
2887
2888                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2889                                 cr   = iptr->sx.s23.s3.c.ref;
2890                                 disp = dseg_add_unique_address(cd, cr);
2891
2892                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
2893                                                                           cr, disp);
2894                         }
2895                         else {
2896                                 disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2897                         }
2898
2899                         /* a1 = classinfo */
2900
2901                         M_ALD(REG_A1, RIP, disp);
2902
2903                         /* a2 = pointer to dimensions = stack pointer */
2904
2905                         M_MOV(REG_SP, REG_A2);
2906
2907                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2908                         M_CALL(REG_ITMP1);
2909
2910                         /* check for exception before result assignment */
2911
2912                         emit_exception_check(cd, iptr);
2913
2914                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2915                         M_INTMOVE(REG_RESULT, s1);
2916                         emit_store_dst(jd, iptr, s1);
2917                         break;
2918
2919                 default:
2920                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2921                                                                                    iptr->opc);
2922                         return false;
2923         } /* switch */
2924
2925         } /* for instruction */
2926
2927         MCODECHECK(512); /* XXX require a lower number? */
2928
2929         /* At the end of a basic block we may have to append some nops,
2930            because the patcher stub calling code might be longer than the
2931            actual instruction. So codepatching does not change the
2932            following block unintentionally. */
2933
2934         if (cd->mcodeptr < cd->lastmcodeptr) {
2935                 while (cd->mcodeptr < cd->lastmcodeptr) {
2936                         M_NOP;
2937                 }
2938         }
2939
2940         } /* if (bptr -> flags >= BBREACHED) */
2941         } /* for basic block */
2942
2943         /* Generate patcher traps. */
2944
2945         emit_patcher_traps(jd);
2946
2947         /* everything's ok */
2948
2949         return true;
2950 }
2951
2952
2953 /* codegen_emit_stub_native ****************************************************
2954
2955    Emits a stub routine which calls a native method.
2956
2957 *******************************************************************************/
2958
2959 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f, int skipparams)
2960 {
2961         methodinfo  *m;
2962         codeinfo    *code;
2963         codegendata *cd;
2964         methoddesc  *md;
2965         int          i, j;
2966         int          s1, s2;
2967         int          disp;
2968
2969         /* Sanity check. */
2970
2971         assert(f != NULL);
2972
2973         /* Get required compiler data. */
2974
2975         m    = jd->m;
2976         code = jd->code;
2977         cd   = jd->cd;
2978
2979         /* initialize variables */
2980
2981         md = m->parseddesc;
2982
2983         /* calculate stack frame size */
2984
2985         cd->stackframesize =
2986                 sizeof(stackframeinfo_t) / SIZEOF_VOID_P +
2987                 sizeof(localref_table) / SIZEOF_VOID_P +
2988                 md->paramcount +
2989                 (md->returntype.type == TYPE_VOID ? 0 : 1) +
2990                 nmd->memuse;
2991
2992         ALIGN_ODD(cd->stackframesize);              /* keep stack 16-byte aligned */
2993
2994         /* create method header */
2995
2996         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2997         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2998         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2999         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
3000         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
3001
3002 #if defined(ENABLE_PROFILING)
3003         /* generate native method profiling code */
3004
3005         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3006                 /* count frequency */
3007
3008                 M_MOV_IMM(code, REG_ITMP3);
3009                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3010         }
3011 #endif
3012
3013         /* generate stub code */
3014
3015         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3016
3017 #if defined(ENABLE_GC_CACAO)
3018         /* Save callee saved integer registers in stackframeinfo (GC may
3019            need to recover them during a collection). */
3020
3021         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
3022                 OFFSET(stackframeinfo_t, intregs);
3023
3024         for (i = 0; i < INT_SAV_CNT; i++)
3025                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3026 #endif
3027
3028         /* save integer and float argument registers */
3029
3030         for (i = 0; i < md->paramcount; i++) {
3031                 if (!md->params[i].inmemory) {
3032                         s1 = md->params[i].regoff;
3033
3034                         switch (md->paramtypes[i].type) {
3035                         case TYPE_INT:
3036                         case TYPE_LNG:
3037                         case TYPE_ADR:
3038                                 M_LST(s1, REG_SP, i * 8);
3039                                 break;
3040                         case TYPE_FLT:
3041                         case TYPE_DBL:
3042                                 M_DST(s1, REG_SP, i * 8);
3043                                 break;
3044                         }
3045                 }
3046         }
3047
3048         /* create dynamic stack info */
3049
3050         M_MOV(REG_SP, REG_A0);
3051         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3052         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3053         M_CALL(REG_ITMP1);
3054
3055         /* remember class argument */
3056
3057         if (m->flags & ACC_STATIC)
3058                 M_MOV(REG_RESULT, REG_ITMP2);
3059
3060         /* restore integer and float argument registers */
3061
3062         for (i = 0; i < md->paramcount; i++) {
3063                 if (!md->params[i].inmemory) {
3064                         s1 = md->params[i].regoff;
3065
3066                         switch (md->paramtypes[i].type) {
3067                         case TYPE_INT:
3068                         case TYPE_LNG:
3069                         case TYPE_ADR:
3070                                 M_LLD(s1, REG_SP, i * 8);
3071                                 break;
3072                         case TYPE_FLT:
3073                         case TYPE_DBL:
3074                                 M_DLD(s1, REG_SP, i * 8);
3075                                 break;
3076                         }
3077                 }
3078         }
3079
3080         /* Copy or spill arguments to new locations. */
3081
3082         for (i = md->paramcount - 1, j = i + skipparams; i >= 0; i--, j--) {
3083                 s2 = nmd->params[j].regoff;
3084
3085                 switch (md->paramtypes[i].type) {
3086                 case TYPE_INT:
3087                 case TYPE_LNG:
3088                 case TYPE_ADR:
3089                         if (!md->params[i].inmemory) {
3090                                 s1 = md->params[i].regoff;
3091
3092                                 if (!nmd->params[j].inmemory)
3093                                         M_INTMOVE(s1, s2);
3094                                 else
3095                                         M_LST(s1, REG_SP, s2);
3096                         }
3097                         else {
3098                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3099                                 M_LLD(REG_ITMP1, REG_SP, s1);
3100                                 M_LST(REG_ITMP1, REG_SP, s2);
3101                         }
3102                         break;
3103                 case TYPE_FLT:
3104                         /* We only copy spilled float arguments, as the float
3105                            argument registers keep unchanged. */
3106
3107                         if (md->params[i].inmemory) {
3108                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3109
3110                                 M_FLD(REG_FTMP1, REG_SP, s1);
3111                                 M_FST(REG_FTMP1, REG_SP, s2);
3112                         }
3113                         break;
3114                 case TYPE_DBL:
3115                         if (md->params[i].inmemory) {
3116                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3117                                 M_DLD(REG_FTMP1, REG_SP, s1);
3118                                 M_DST(REG_FTMP1, REG_SP, s2);
3119                         }
3120                         break;
3121                 }
3122         }
3123
3124         /* Handle native Java methods. */
3125
3126         if (m->flags & ACC_NATIVE) {
3127                 /* put class into second argument register */
3128
3129                 if (m->flags & ACC_STATIC)
3130                         M_MOV(REG_ITMP2, REG_A1);
3131
3132                 /* put env into first argument register */
3133
3134                 M_MOV_IMM(VM_get_jnienv(), REG_A0);
3135         }
3136
3137         /* Call the native function. */
3138
3139         disp = dseg_add_functionptr(cd, f);
3140         M_ALD(REG_ITMP1, RIP, disp);
3141         M_CALL(REG_ITMP1);
3142
3143         /* save return value */
3144
3145         switch (md->returntype.type) {
3146         case TYPE_INT:
3147         case TYPE_LNG:
3148         case TYPE_ADR:
3149                 switch (md->returntype.primitivetype) {
3150                 case PRIMITIVETYPE_BOOLEAN:
3151                         M_BZEXT(REG_RESULT, REG_RESULT);
3152                         break;
3153                 case PRIMITIVETYPE_BYTE:
3154                         M_BSEXT(REG_RESULT, REG_RESULT);
3155                         break;
3156                 case PRIMITIVETYPE_CHAR:
3157                         M_CZEXT(REG_RESULT, REG_RESULT);
3158                         break;
3159                 case PRIMITIVETYPE_SHORT:
3160                         M_SSEXT(REG_RESULT, REG_RESULT);
3161                         break;
3162                 }
3163                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3164                 break;
3165         case TYPE_FLT:
3166         case TYPE_DBL:
3167                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3168                 break;
3169         case TYPE_VOID:
3170                 break;
3171         }
3172
3173         /* remove native stackframe info */
3174
3175         M_MOV(REG_SP, REG_A0);
3176         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3177         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3178         M_CALL(REG_ITMP1);
3179         M_MOV(REG_RESULT, REG_ITMP3);
3180
3181         /* restore return value */
3182
3183         switch (md->returntype.type) {
3184         case TYPE_INT:
3185         case TYPE_LNG:
3186         case TYPE_ADR:
3187                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3188                 break;
3189         case TYPE_FLT:
3190         case TYPE_DBL:
3191                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3192                 break;
3193         case TYPE_VOID:
3194                 break;
3195         }
3196
3197 #if defined(ENABLE_GC_CACAO)
3198         /* Restore callee saved integer registers from stackframeinfo (GC
3199            might have modified them during a collection). */
3200
3201         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
3202                 OFFSET(stackframeinfo_t, intregs);
3203
3204         for (i = 0; i < INT_SAV_CNT; i++)
3205                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3206 #endif
3207
3208         /* remove stackframe */
3209
3210         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3211
3212         /* test for exception */
3213
3214         M_TEST(REG_ITMP3);
3215         M_BNE(1);
3216         M_RET;
3217
3218         /* handle exception */
3219
3220         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3221         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3222         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3223
3224         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3225         M_JMP(REG_ITMP3);
3226 }
3227
3228
3229 /*
3230  * These are local overrides for various environment variables in Emacs.
3231  * Please do not remove this and leave it at the end of the file, where
3232  * Emacs will automagically detect them.
3233  * ---------------------------------------------------------------------
3234  * Local variables:
3235  * mode: c
3236  * indent-tabs-mode: t
3237  * c-basic-offset: 4
3238  * tab-width: 4
3239  * End:
3240  * vim:noexpandtab:sw=4:ts=4:
3241  */