src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25 */
  26
  27
  28 #include "config.h"
  29
  30 #include <assert.h>
  31 #include <stdio.h>
  32 #include <stdint.h>
  33
  34 #include "vm/types.h"
  35
  36 #include "md-abi.h"
  37
  38 #include "vm/jit/x86_64/arch.h"
  39 #include "vm/jit/x86_64/codegen.h"
  40 #include "vm/jit/x86_64/emit.h"
  41
  42 #include "mm/memory.h"
  43
  44 #include "native/jni.h"
  45 #include "native/localref.h"
  46 #include "native/native.h"
  47
  48 #include "threads/lock-common.h"
  49
  50 #include "vm/builtin.h"
  51 #include "vm/exceptions.h"
  52 #include "vm/global.h"
  53 #include "vm/stringlocal.h"
  54 #include "vm/vm.h"
  55
  56 #include "vm/jit/abi.h"
  57 #include "vm/jit/asmpart.h"
  58 #include "vm/jit/code.h"
  59 #include "vm/jit/codegen-common.h"
  60 #include "vm/jit/dseg.h"
  61 #include "vm/jit/emit-common.h"
  62 #include "vm/jit/jit.h"
  63 #include "vm/jit/methodheader.h"
  64 #include "vm/jit/parse.h"
  65 #include "vm/jit/patcher-common.h"
  66 #include "vm/jit/reg.h"
  67 #include "vm/jit/replace.h"
  68 #include "vm/jit/stacktrace.h"
  69
  70 #if defined(ENABLE_LSRA)
  71 # include "vm/jit/allocator/lsra.h"
  72 #endif
  73
  74 #include "vmcore/loader.h"
  75 #include "vmcore/options.h"
  76 #include "vmcore/statistics.h"
  77
  78
  79 /* codegen_emit ****************************************************************
  80
  81    Generates machine code.
  82
  83 *******************************************************************************/
  84
  85 bool codegen_emit(jitdata *jd)
  86 {
  87         methodinfo         *m;
  88         codeinfo           *code;
  89         codegendata        *cd;
  90         registerdata       *rd;
  91         s4                  len, s1, s2, s3, d, disp;
  92         u2                  currentline;
  93         ptrint              a;
  94         varinfo            *var, *dst;
  95         basicblock         *bptr;
  96         instruction        *iptr;
  97         exception_entry    *ex;
  98         constant_classref  *cr;
  99         unresolved_class   *uc;
 100         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 101         unresolved_method  *um;
 102         builtintable_entry *bte;
 103         methoddesc         *md;
 104         fieldinfo          *fi;
 105         unresolved_field   *uf;
 106         s4                  fieldtype;
 107         s4                 varindex;
 108
 109         /* get required compiler data */
 110
 111         m    = jd->m;
 112         code = jd->code;
 113         cd   = jd->cd;
 114         rd   = jd->rd;
 115
 116         /* prevent compiler warnings */
 117
 118         d   = 0;
 119         lm  = NULL;
 120         um  = NULL;
 121         bte = NULL;
 122
 123         {
 124         s4 i, p, t, l;
 125         s4 savedregs_num;
 126
 127         savedregs_num = 0;
 128
 129         /* space to save used callee saved registers */
 130
 131         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 132         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 133
 134         cd->stackframesize = rd->memuse + savedregs_num;
 135
 136 #if defined(ENABLE_THREADS)
 137         /* space to save argument of monitor_enter */
 138
 139         if (checksync && code_is_synchronized(code))
 140                 cd->stackframesize++;
 141 #endif
 142
 143         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 144            native code e.g. libc or jni (alignment problems with
 145            movaps). */
 146
 147         if (!code_is_leafmethod(code) || opt_verbosecall)
 148                 cd->stackframesize |= 0x1;
 149
 150         /* create method header */
 151
 152         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 153         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 154
 155         /* IsSync contains the offset relative to the stack pointer for the
 156            argument of monitor_exit used in the exception handler. Since the
 157            offset could be zero and give a wrong meaning of the flag it is
 158            offset by one.
 159         */
 160         /* XXX Remove this "offset by one". */
 161
 162         code->synchronizedoffset = (rd->memuse + 1) * 8;
 163
 164         if (code_is_leafmethod(code))
 165                 (void) dseg_add_unique_s4(cd, 1);                  /* IsLeaf          */
 166         else
 167                 (void) dseg_add_unique_s4(cd, 0);                  /* IsLeaf          */
 168
 169         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 170         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 171
 172         (void) dseg_addlinenumbertablesize(cd);
 173
 174         (void) dseg_add_unique_s4(cd, jd->exceptiontablelength); /* ExTableSize   */
 175
 176         /* create exception table */
 177
 178         for (ex = jd->exceptiontable; ex != NULL; ex = ex->down) {
 179                 dseg_add_target(cd, ex->start);
 180                 dseg_add_target(cd, ex->end);
 181                 dseg_add_target(cd, ex->handler);
 182                 (void) dseg_add_unique_address(cd, ex->catchtype.any);
 183         }
 184
 185 #if defined(ENABLE_PROFILING)
 186         /* generate method profiling code */
 187
 188         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 189                 /* count frequency */
 190
 191                 M_MOV_IMM(code, REG_ITMP3);
 192                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 193
 194                 PROFILE_CYCLE_START;
 195         }
 196 #endif
 197
 198         /* create stack frame (if necessary) */
 199
 200         if (cd->stackframesize)
 201                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 202
 203         /* save used callee saved registers */
 204
 205         p = cd->stackframesize;
 206         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 207                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 208         }
 209         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 210                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 211         }
 212
 213         /* take arguments out of register or stack frame */
 214
 215         md = m->parseddesc;
 216
 217         for (p = 0, l = 0; p < md->paramcount; p++) {
 218                 t = md->paramtypes[p].type;
 219
 220                 varindex = jd->local_map[l * 5 + t];
 221
 222                 l++;
 223                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 224                         l++;
 225
 226                 if (varindex == UNUSED)
 227                         continue;
 228
 229                 var = VAR(varindex);
 230
 231                 s1 = md->params[p].regoff;
 232
 233                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 234                         if (!md->params[p].inmemory) {           /* register arguments    */
 235                                 if (!IS_INMEMORY(var->flags))
 236                                         M_INTMOVE(s1, var->vv.regoff);
 237                                 else
 238                                     M_LST(s1, REG_SP, var->vv.regoff);
 239                         }
 240                         else {                                 /* stack arguments       */
 241                                 if (!IS_INMEMORY(var->flags))
 242                                         /* + 8 for return address */
 243                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 244                                 else
 245                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 246                         }
 247                 }
 248                 else {                                     /* floating args         */
 249                         if (!md->params[p].inmemory) {           /* register arguments    */
 250                                 if (!IS_INMEMORY(var->flags))
 251                                         M_FLTMOVE(s1, var->vv.regoff);
 252                                 else
 253                                         M_DST(s1, REG_SP, var->vv.regoff);
 254                         }
 255                         else {                                 /* stack arguments       */
 256                                 if (!IS_INMEMORY(var->flags))
 257                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 258                                 else
 259                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 260                         }
 261                 }
 262         }
 263
 264         /* save monitorenter argument */
 265
 266 #if defined(ENABLE_THREADS)
 267         if (checksync && code_is_synchronized(code)) {
 268                 /* stack offset for monitor argument */
 269
 270                 s1 = rd->memuse;
 271
 272                 if (opt_verbosecall) {
 273                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 274
 275                         for (p = 0; p < INT_ARG_CNT; p++)
 276                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 277
 278                         for (p = 0; p < FLT_ARG_CNT; p++)
 279                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 280
 281                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 282                 }
 283
 284                 /* decide which monitor enter function to call */
 285
 286                 if (m->flags & ACC_STATIC) {
 287                         M_MOV_IMM(&m->class->object.header, REG_A0);
 288                 }
 289                 else {
 290                         M_TEST(REG_A0);
 291                         M_BNE(8);
 292                         M_ALD_MEM(REG_A0, EXCEPTION_HARDWARE_NULLPOINTER);
 293                 }
 294
 295                 M_AST(REG_A0, REG_SP, s1 * 8);
 296                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 297                 M_CALL(REG_ITMP1);
 298
 299                 if (opt_verbosecall) {
 300                         for (p = 0; p < INT_ARG_CNT; p++)
 301                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 302
 303                         for (p = 0; p < FLT_ARG_CNT; p++)
 304                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 305
 306                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 307                 }
 308         }
 309 #endif
 310
 311 #if !defined(NDEBUG)
 312         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 313                 emit_verbosecall_enter(jd);
 314 #endif /* !defined(NDEBUG) */
 315
 316         }
 317
 318         /* end of header generation */
 319
 320         /* create replacement points */
 321
 322         REPLACEMENT_POINTS_INIT(cd, jd);
 323
 324         /* walk through all basic blocks */
 325
 326         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 327
 328                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 329
 330                 if (bptr->flags >= BBREACHED) {
 331
 332                 /* branch resolving */
 333
 334                 codegen_resolve_branchrefs(cd, bptr);
 335
 336                 /* handle replacement points */
 337
 338                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 339
 340                 /* copy interface registers to their destination */
 341
 342                 len = bptr->indepth;
 343                 MCODECHECK(512);
 344
 345 #if defined(ENABLE_PROFILING)
 346                 /* generate basicblock profiling code */
 347
 348                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 349                         /* count frequency */
 350
 351                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 352                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 353
 354                         /* if this is an exception handler, start profiling again */
 355
 356                         if (bptr->type == BBTYPE_EXH)
 357                                 PROFILE_CYCLE_START;
 358                 }
 359 #endif
 360
 361 #if defined(ENABLE_LSRA)
 362                 if (opt_lsra) {
 363                         while (len) {
 364                                 len--;
 365                                 src = bptr->invars[len];
 366                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 367                                         if (bptr->type == BBTYPE_EXH) {
 368 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 369                                                 if (!IS_INMEMORY(src->flags))
 370                                                         d= src->vv.regoff;
 371                                                 else
 372                                                         d=REG_ITMP1;
 373                                                 M_INTMOVE(REG_ITMP1, d);
 374                                                 emit_store(jd, NULL, src, d);
 375                                         }
 376                                 }
 377                         }
 378
 379                 } else {
 380 #endif
 381
 382                 while (len) {
 383                         len--;
 384                         var = VAR(bptr->invars[len]);
 385                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 386                                 if (bptr->type == BBTYPE_EXH) {
 387                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 388                                         M_INTMOVE(REG_ITMP1, d);
 389                                         emit_store(jd, NULL, var, d);
 390                                 }
 391                         }
 392                         else {
 393                                 assert((var->flags & INOUT));
 394                         }
 395                 }
 396 #if defined(ENABLE_LSRA)
 397                 }
 398 #endif
 399                 /* walk through all instructions */
 400
 401                 len = bptr->icount;
 402                 currentline = 0;
 403
 404                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 405                         if (iptr->line != currentline) {
 406                                 dseg_addlinenumber(cd, iptr->line);
 407                                 currentline = iptr->line;
 408                         }
 409
 410                         MCODECHECK(1024);                         /* 1KB should be enough */
 411
 412                 switch (iptr->opc) {
 413                 case ICMD_NOP:        /* ...  ==> ...                                 */
 414                 case ICMD_POP:        /* ..., value  ==> ...                          */
 415                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 416                         break;
 417
 418                 case ICMD_INLINE_START:
 419
 420                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 421                         break;
 422
 423                 case ICMD_INLINE_BODY:
 424
 425                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 426                         dseg_addlinenumber_inline_start(cd, iptr);
 427                         dseg_addlinenumber(cd, iptr->line);
 428                         break;
 429
 430                 case ICMD_INLINE_END:
 431
 432                         dseg_addlinenumber_inline_end(cd, iptr);
 433                         dseg_addlinenumber(cd, iptr->line);
 434                         break;
 435
 436                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 437
 438                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 439                         emit_nullpointer_check(cd, iptr, s1);
 440                         break;
 441
 442                 /* constant operations ************************************************/
 443
 444                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 445
 446                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 447                         ICONST(d, iptr->sx.val.i);
 448                         emit_store_dst(jd, iptr, d);
 449                         break;
 450
 451                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 452
 453                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 454                         LCONST(d, iptr->sx.val.l);
 455                         emit_store_dst(jd, iptr, d);
 456                         break;
 457
 458                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 459
 460                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 461                         disp = dseg_add_float(cd, iptr->sx.val.f);
 462                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 463                         emit_store_dst(jd, iptr, d);
 464                         break;
 465
 466                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 467
 468                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 469                         disp = dseg_add_double(cd, iptr->sx.val.d);
 470                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 471                         emit_store_dst(jd, iptr, d);
 472                         break;
 473
 474                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 475
 476                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 477
 478                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 479                                 cr   = iptr->sx.val.c.ref;
 480                                 disp = dseg_add_unique_address(cd, cr);
 481
 482 /*                              PROFILE_CYCLE_STOP; */
 483
 484                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
 485                                                                           cr, disp);
 486
 487 /*                              PROFILE_CYCLE_START; */
 488
 489                                 M_ALD(d, RIP, disp);
 490                         }
 491                         else {
 492                                 if (iptr->sx.val.anyptr == 0) {
 493                                         M_CLR(d);
 494                                 }
 495                                 else {
 496                                         disp = dseg_add_address(cd, iptr->sx.val.anyptr);
 497                                         M_ALD(d, RIP, disp);
 498                                 }
 499                         }
 500                         emit_store_dst(jd, iptr, d);
 501                         break;
 502
 503
 504                 /* load/store/copy/move operations ************************************/
 505
 506                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 507                 case ICMD_ALOAD:      /* s1 = local variable                          */
 508                 case ICMD_LLOAD:
 509                 case ICMD_FLOAD:
 510                 case ICMD_DLOAD:
 511                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 512                 case ICMD_LSTORE:
 513                 case ICMD_FSTORE:
 514                 case ICMD_DSTORE:
 515                 case ICMD_COPY:
 516                 case ICMD_MOVE:
 517
 518                         emit_copy(jd, iptr);
 519                         break;
 520
 521                 case ICMD_ASTORE:
 522                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 523                                 emit_copy(jd, iptr);
 524                         break;
 525
 526                 /* integer operations *************************************************/
 527
 528                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 529
 530                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 531                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 532                         M_INTMOVE(s1, d);
 533                         M_INEG(d);
 534                         emit_store_dst(jd, iptr, d);
 535                         break;
 536
 537                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 538
 539                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 540                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 541                         M_INTMOVE(s1, d);
 542                         M_LNEG(d);
 543                         emit_store_dst(jd, iptr, d);
 544                         break;
 545
 546                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 547
 548                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 549                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 550                         M_ISEXT(s1, d);
 551                         emit_store_dst(jd, iptr, d);
 552                         break;
 553
 554                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 555
 556                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 557                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 558                         M_IMOV(s1, d);
 559                         emit_store_dst(jd, iptr, d);
 560                         break;
 561
 562                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 563
 564                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 565                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 566                         M_BSEXT(s1, d);
 567                         emit_store_dst(jd, iptr, d);
 568                         break;
 569
 570                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 571
 572                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 573                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 574                         M_CZEXT(s1, d);
 575                         emit_store_dst(jd, iptr, d);
 576                         break;
 577
 578                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 579
 580                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 581                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 582                         M_SSEXT(s1, d);
 583                         emit_store_dst(jd, iptr, d);
 584                         break;
 585
 586
 587                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 588
 589                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 590                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 591                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 592                         if (s2 == d)
 593                                 M_IADD(s1, d);
 594                         else {
 595                                 M_INTMOVE(s1, d);
 596                                 M_IADD(s2, d);
 597                         }
 598                         emit_store_dst(jd, iptr, d);
 599                         break;
 600
 601                 case ICMD_IINC:
 602                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 603                                       /* sx.val.i = constant                             */
 604
 605                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 606                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 607
 608                         /* Using inc and dec is not faster than add (tested with
 609                            sieve). */
 610
 611                         M_INTMOVE(s1, d);
 612                         M_IADD_IMM(iptr->sx.val.i, d);
 613                         emit_store_dst(jd, iptr, d);
 614                         break;
 615
 616                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 617
 618                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 619                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 620                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 621                         if (s2 == d)
 622                                 M_LADD(s1, d);
 623                         else {
 624                                 M_INTMOVE(s1, d);
 625                                 M_LADD(s2, d);
 626                         }
 627                         emit_store_dst(jd, iptr, d);
 628                         break;
 629
 630                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 631                                       /* sx.val.l = constant                             */
 632
 633                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 634                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 635                         M_INTMOVE(s1, d);
 636                         if (IS_IMM32(iptr->sx.val.l))
 637                                 M_LADD_IMM(iptr->sx.val.l, d);
 638                         else {
 639                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 640                                 M_LADD(REG_ITMP2, d);
 641                         }
 642                         emit_store_dst(jd, iptr, d);
 643                         break;
 644
 645                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 646
 647                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 648                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 649                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 650                         if (s2 == d) {
 651                                 M_INTMOVE(s1, REG_ITMP1);
 652                                 M_ISUB(s2, REG_ITMP1);
 653                                 M_INTMOVE(REG_ITMP1, d);
 654                         } else {
 655                                 M_INTMOVE(s1, d);
 656                                 M_ISUB(s2, d);
 657                         }
 658                         emit_store_dst(jd, iptr, d);
 659                         break;
 660
 661                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 662                                       /* sx.val.i = constant                             */
 663
 664                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 665                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 666                         M_INTMOVE(s1, d);
 667                         M_ISUB_IMM(iptr->sx.val.i, d);
 668                         emit_store_dst(jd, iptr, d);
 669                         break;
 670
 671                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 672
 673                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 674                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 675                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 676                         if (s2 == d) {
 677                                 M_INTMOVE(s1, REG_ITMP1);
 678                                 M_LSUB(s2, REG_ITMP1);
 679                                 M_INTMOVE(REG_ITMP1, d);
 680                         } else {
 681                                 M_INTMOVE(s1, d);
 682                                 M_LSUB(s2, d);
 683                         }
 684                         emit_store_dst(jd, iptr, d);
 685                         break;
 686
 687                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 688                                       /* sx.val.l = constant                             */
 689
 690                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 691                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 692                         M_INTMOVE(s1, d);
 693                         if (IS_IMM32(iptr->sx.val.l))
 694                                 M_LSUB_IMM(iptr->sx.val.l, d);
 695                         else {
 696                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 697                                 M_LSUB(REG_ITMP2, d);
 698                         }
 699                         emit_store_dst(jd, iptr, d);
 700                         break;
 701
 702                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 703
 704                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 705                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 706                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 707                         if (s2 == d)
 708                                 M_IMUL(s1, d);
 709                         else {
 710                                 M_INTMOVE(s1, d);
 711                                 M_IMUL(s2, d);
 712                         }
 713                         emit_store_dst(jd, iptr, d);
 714                         break;
 715
 716                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 717                                       /* sx.val.i = constant                             */
 718
 719                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 720                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 721                         if (iptr->sx.val.i == 2) {
 722                                 M_INTMOVE(s1, d);
 723                                 M_ISLL_IMM(1, d);
 724                         } else
 725                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 726                         emit_store_dst(jd, iptr, d);
 727                         break;
 728
 729                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 730
 731                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 732                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 733                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 734                         if (s2 == d)
 735                                 M_LMUL(s1, d);
 736                         else {
 737                                 M_INTMOVE(s1, d);
 738                                 M_LMUL(s2, d);
 739                         }
 740                         emit_store_dst(jd, iptr, d);
 741                         break;
 742
 743                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 744                                       /* sx.val.l = constant                             */
 745
 746                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 747                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 748                         if (IS_IMM32(iptr->sx.val.l))
 749                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 750                         else {
 751                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 752                                 M_INTMOVE(s1, d);
 753                                 M_LMUL(REG_ITMP2, d);
 754                         }
 755                         emit_store_dst(jd, iptr, d);
 756                         break;
 757
 758                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 759
 760                         s1 = emit_load_s1(jd, iptr, RAX);
 761                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 762                         d = codegen_reg_of_dst(jd, iptr, RAX);
 763
 764                         M_INTMOVE(s1, RAX);
 765                         M_INTMOVE(s2, REG_ITMP3);
 766                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 767
 768                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 769
 770                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 771                         M_BNE(4 + 6);
 772                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 773                         M_BEQ(1 + 3);                                          /* 6 bytes */
 774
 775                         emit_cltd(cd);                                         /* 1 byte  */
 776                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 777
 778                         M_INTMOVE(RAX, d);
 779                         emit_store_dst(jd, iptr, d);
 780                         dst = VAROP(iptr->dst);
 781                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 782                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 783                         break;
 784
 785                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 786
 787                         s1 = emit_load_s1(jd, iptr, RAX);
 788                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 789                         d = codegen_reg_of_dst(jd, iptr, RDX);
 790
 791                         M_INTMOVE(s1, RAX);
 792                         M_INTMOVE(s2, REG_ITMP3);
 793                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 794
 795                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 796
 797                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 798                         M_BNE(3 + 4 + 6);
 799                         M_CLR(RDX);                                            /* 3 bytes */
 800                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 801                         M_BEQ(1 + 3);                                          /* 6 bytes */
 802
 803                         emit_cltd(cd);                                         /* 1 byte  */
 804                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 805
 806                         M_INTMOVE(RDX, d);
 807                         emit_store_dst(jd, iptr, d);
 808                         dst = VAROP(iptr->dst);
 809                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 810                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 811                         break;
 812
 813                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 814                                       /* sx.val.i = constant                             */
 815
 816                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 817                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 818                         M_INTMOVE(s1, REG_ITMP1);
 819                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 820                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 821                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 822                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 823                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 824                         emit_store_dst(jd, iptr, d);
 825                         break;
 826
 827                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 828                                       /* sx.val.i = constant                             */
 829
 830                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 831                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 832                         M_INTMOVE(s1, REG_ITMP1);
 833                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 834                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 835                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 836                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 837                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 838                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 839                         emit_store_dst(jd, iptr, d);
 840                         break;
 841
 842
 843                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 844
 845                         s1 = emit_load_s1(jd, iptr, RAX);
 846                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 847                         d = codegen_reg_of_dst(jd, iptr, RAX);
 848
 849                         M_INTMOVE(s1, RAX);
 850                         M_INTMOVE(s2, REG_ITMP3);
 851                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 852
 853                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 854
 855                         /* check as described in jvm spec */
 856                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 857                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 858                         M_BNE(4 + 6);
 859                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 860                         M_BEQ(2 + 3);                                          /* 6 bytes */
 861
 862                         emit_cqto(cd);                                         /* 2 bytes */
 863                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 864
 865                         M_INTMOVE(RAX, d);
 866                         emit_store_dst(jd, iptr, d);
 867                         dst = VAROP(iptr->dst);
 868                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 869                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 870                         break;
 871
 872                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 873
 874                         s1 = emit_load_s1(jd, iptr, RAX);
 875                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 876                         d = codegen_reg_of_dst(jd, iptr, RDX);
 877
 878                         M_INTMOVE(s1, RAX);
 879                         M_INTMOVE(s2, REG_ITMP3);
 880                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 881
 882                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 883
 884                         /* check as described in jvm spec */
 885                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 886                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 887                         M_BNE(3 + 4 + 6);
 888                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 889                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 890                         M_BEQ(2 + 3);                                          /* 6 bytes */
 891
 892                         emit_cqto(cd);                                         /* 2 bytes */
 893                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 894
 895                         M_INTMOVE(RDX, d);
 896                         emit_store_dst(jd, iptr, d);
 897                         dst = VAROP(iptr->dst);
 898                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 899                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 900                         break;
 901
 902                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 903                                       /* sx.val.i = constant                             */
 904
 905                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 906                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 907                         M_INTMOVE(s1, REG_ITMP1);
 908                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 909                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 910                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 911                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 912                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 913                         emit_store_dst(jd, iptr, d);
 914                         break;
 915
 916                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 917                                       /* sx.val.l = constant                             */
 918
 919                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 920                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 921                         M_INTMOVE(s1, REG_ITMP1);
 922                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 923                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 924                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 925                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 926                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 927                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 928                         emit_store_dst(jd, iptr, d);
 929                         break;
 930
 931                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 932
 933                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 934                         emit_ishift(jd, SHIFT_SHL, iptr);
 935                         break;
 936
 937                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 938                                       /* sx.val.i = constant                             */
 939
 940                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 941                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 942                         M_INTMOVE(s1, d);
 943                         M_ISLL_IMM(iptr->sx.val.i, d);
 944                         emit_store_dst(jd, iptr, d);
 945                         break;
 946
 947                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 948
 949                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 950                         emit_ishift(jd, SHIFT_SAR, iptr);
 951                         break;
 952
 953                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 954                                       /* sx.val.i = constant                             */
 955
 956                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 957                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 958                         M_INTMOVE(s1, d);
 959                         M_ISRA_IMM(iptr->sx.val.i, d);
 960                         emit_store_dst(jd, iptr, d);
 961                         break;
 962
 963                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 964
 965                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 966                         emit_ishift(jd, SHIFT_SHR, iptr);
 967                         break;
 968
 969                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 970                                       /* sx.val.i = constant                             */
 971
 972                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 973                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 974                         M_INTMOVE(s1, d);
 975                         M_ISRL_IMM(iptr->sx.val.i, d);
 976                         emit_store_dst(jd, iptr, d);
 977                         break;
 978
 979                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 980
 981                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 982                         emit_lshift(jd, SHIFT_SHL, iptr);
 983                         break;
 984
 985         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 986                                           /* sx.val.i = constant                             */
 987
 988                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 989                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 990                         M_INTMOVE(s1, d);
 991                         M_LSLL_IMM(iptr->sx.val.i, d);
 992                         emit_store_dst(jd, iptr, d);
 993                         break;
 994
 995                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 996
 997                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 998                         emit_lshift(jd, SHIFT_SAR, iptr);
 999                         break;
1000
1001                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1002                                       /* sx.val.i = constant                             */
1003
1004                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1005                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1006                         M_INTMOVE(s1, d);
1007                         M_LSRA_IMM(iptr->sx.val.i, d);
1008                         emit_store_dst(jd, iptr, d);
1009                         break;
1010
1011                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1012
1013                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1014                         emit_lshift(jd, SHIFT_SHR, iptr);
1015                         break;
1016
1017                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1018                                       /* sx.val.l = constant                             */
1019
1020                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1021                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1022                         M_INTMOVE(s1, d);
1023                         M_LSRL_IMM(iptr->sx.val.i, d);
1024                         emit_store_dst(jd, iptr, d);
1025                         break;
1026
1027                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1028
1029                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1030                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1031                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1032                         if (s2 == d)
1033                                 M_IAND(s1, d);
1034                         else {
1035                                 M_INTMOVE(s1, d);
1036                                 M_IAND(s2, d);
1037                         }
1038                         emit_store_dst(jd, iptr, d);
1039                         break;
1040
1041                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1042                                       /* sx.val.i = constant                             */
1043
1044                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1045                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1046                         M_INTMOVE(s1, d);
1047                         M_IAND_IMM(iptr->sx.val.i, d);
1048                         emit_store_dst(jd, iptr, d);
1049                         break;
1050
1051                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1052
1053                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1054                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1055                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1056                         if (s2 == d)
1057                                 M_LAND(s1, d);
1058                         else {
1059                                 M_INTMOVE(s1, d);
1060                                 M_LAND(s2, d);
1061                         }
1062                         emit_store_dst(jd, iptr, d);
1063                         break;
1064
1065                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1066                                       /* sx.val.l = constant                             */
1067
1068                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1069                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1070                         M_INTMOVE(s1, d);
1071                         if (IS_IMM32(iptr->sx.val.l))
1072                                 M_LAND_IMM(iptr->sx.val.l, d);
1073                         else {
1074                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1075                                 M_LAND(REG_ITMP2, d);
1076                         }
1077                         emit_store_dst(jd, iptr, d);
1078                         break;
1079
1080                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1081
1082                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1083                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1084                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1085                         if (s2 == d)
1086                                 M_IOR(s1, d);
1087                         else {
1088                                 M_INTMOVE(s1, d);
1089                                 M_IOR(s2, d);
1090                         }
1091                         emit_store_dst(jd, iptr, d);
1092                         break;
1093
1094                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1095                                       /* sx.val.i = constant                             */
1096
1097                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1098                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1099                         M_INTMOVE(s1, d);
1100                         M_IOR_IMM(iptr->sx.val.i, d);
1101                         emit_store_dst(jd, iptr, d);
1102                         break;
1103
1104                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1105
1106                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1107                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1108                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1109                         if (s2 == d)
1110                                 M_LOR(s1, d);
1111                         else {
1112                                 M_INTMOVE(s1, d);
1113                                 M_LOR(s2, d);
1114                         }
1115                         emit_store_dst(jd, iptr, d);
1116                         break;
1117
1118                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1119                                       /* sx.val.l = constant                             */
1120
1121                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1122                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1123                         M_INTMOVE(s1, d);
1124                         if (IS_IMM32(iptr->sx.val.l))
1125                                 M_LOR_IMM(iptr->sx.val.l, d);
1126                         else {
1127                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1128                                 M_LOR(REG_ITMP2, d);
1129                         }
1130                         emit_store_dst(jd, iptr, d);
1131                         break;
1132
1133                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1134
1135                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1136                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1137                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1138                         if (s2 == d)
1139                                 M_IXOR(s1, d);
1140                         else {
1141                                 M_INTMOVE(s1, d);
1142                                 M_IXOR(s2, d);
1143                         }
1144                         emit_store_dst(jd, iptr, d);
1145                         break;
1146
1147                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1148                                       /* sx.val.i = constant                             */
1149
1150                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1151                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1152                         M_INTMOVE(s1, d);
1153                         M_IXOR_IMM(iptr->sx.val.i, d);
1154                         emit_store_dst(jd, iptr, d);
1155                         break;
1156
1157                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1158
1159                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1160                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1161                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1162                         if (s2 == d)
1163                                 M_LXOR(s1, d);
1164                         else {
1165                                 M_INTMOVE(s1, d);
1166                                 M_LXOR(s2, d);
1167                         }
1168                         emit_store_dst(jd, iptr, d);
1169                         break;
1170
1171                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1172                                       /* sx.val.l = constant                             */
1173
1174                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1175                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1176                         M_INTMOVE(s1, d);
1177                         if (IS_IMM32(iptr->sx.val.l))
1178                                 M_LXOR_IMM(iptr->sx.val.l, d);
1179                         else {
1180                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1181                                 M_LXOR(REG_ITMP2, d);
1182                         }
1183                         emit_store_dst(jd, iptr, d);
1184                         break;
1185
1186
1187                 /* floating operations ************************************************/
1188
1189                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1190
1191                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1192                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1193                         disp = dseg_add_s4(cd, 0x80000000);
1194                         M_FLTMOVE(s1, d);
1195                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1196                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1197                         emit_store_dst(jd, iptr, d);
1198                         break;
1199
1200                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1201
1202                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1203                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1204                         disp = dseg_add_s8(cd, 0x8000000000000000);
1205                         M_FLTMOVE(s1, d);
1206                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1207                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1208                         emit_store_dst(jd, iptr, d);
1209                         break;
1210
1211                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1212
1213                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1214                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1215                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1216                         if (s2 == d)
1217                                 M_FADD(s1, d);
1218                         else {
1219                                 M_FLTMOVE(s1, d);
1220                                 M_FADD(s2, d);
1221                         }
1222                         emit_store_dst(jd, iptr, d);
1223                         break;
1224
1225                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1226
1227                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1228                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1229                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1230                         if (s2 == d)
1231                                 M_DADD(s1, d);
1232                         else {
1233                                 M_FLTMOVE(s1, d);
1234                                 M_DADD(s2, d);
1235                         }
1236                         emit_store_dst(jd, iptr, d);
1237                         break;
1238
1239                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1240
1241                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1242                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1243                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1244                         if (s2 == d) {
1245                                 M_FLTMOVE(s2, REG_FTMP2);
1246                                 s2 = REG_FTMP2;
1247                         }
1248                         M_FLTMOVE(s1, d);
1249                         M_FSUB(s2, d);
1250                         emit_store_dst(jd, iptr, d);
1251                         break;
1252
1253                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1254
1255                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1256                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1257                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1258                         if (s2 == d) {
1259                                 M_FLTMOVE(s2, REG_FTMP2);
1260                                 s2 = REG_FTMP2;
1261                         }
1262                         M_FLTMOVE(s1, d);
1263                         M_DSUB(s2, d);
1264                         emit_store_dst(jd, iptr, d);
1265                         break;
1266
1267                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1268
1269                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1270                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1271                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1272                         if (s2 == d)
1273                                 M_FMUL(s1, d);
1274                         else {
1275                                 M_FLTMOVE(s1, d);
1276                                 M_FMUL(s2, d);
1277                         }
1278                         emit_store_dst(jd, iptr, d);
1279                         break;
1280
1281                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1282
1283                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1284                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1285                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1286                         if (s2 == d)
1287                                 M_DMUL(s1, d);
1288                         else {
1289                                 M_FLTMOVE(s1, d);
1290                                 M_DMUL(s2, d);
1291                         }
1292                         emit_store_dst(jd, iptr, d);
1293                         break;
1294
1295                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1296
1297                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1298                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1299                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1300                         if (s2 == d) {
1301                                 M_FLTMOVE(s2, REG_FTMP2);
1302                                 s2 = REG_FTMP2;
1303                         }
1304                         M_FLTMOVE(s1, d);
1305                         M_FDIV(s2, d);
1306                         emit_store_dst(jd, iptr, d);
1307                         break;
1308
1309                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1310
1311                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1312                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1313                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1314                         if (s2 == d) {
1315                                 M_FLTMOVE(s2, REG_FTMP2);
1316                                 s2 = REG_FTMP2;
1317                         }
1318                         M_FLTMOVE(s1, d);
1319                         M_DDIV(s2, d);
1320                         emit_store_dst(jd, iptr, d);
1321                         break;
1322
1323                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1324
1325                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1326                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1327                         M_CVTIF(s1, d);
1328                         emit_store_dst(jd, iptr, d);
1329                         break;
1330
1331                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1332
1333                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1334                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1335                         M_CVTID(s1, d);
1336                         emit_store_dst(jd, iptr, d);
1337                         break;
1338
1339                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1340
1341                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1342                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1343                         M_CVTLF(s1, d);
1344                         emit_store_dst(jd, iptr, d);
1345                         break;
1346
1347                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1348
1349                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1350                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1351                         M_CVTLD(s1, d);
1352                         emit_store_dst(jd, iptr, d);
1353                         break;
1354
1355                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1356
1357                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1358                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1359                         M_CVTFI(s1, d);
1360                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1361                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1362                                 ((REG_RESULT == d) ? 0 : 3);
1363                         M_BNE(disp);
1364                         M_FLTMOVE(s1, REG_FTMP1);
1365                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1366                         M_CALL(REG_ITMP2);
1367                         M_INTMOVE(REG_RESULT, d);
1368                         emit_store_dst(jd, iptr, d);
1369                         break;
1370
1371                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1372
1373                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1374                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1375                         M_CVTDI(s1, d);
1376                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1377                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1378                                 ((REG_RESULT == d) ? 0 : 3);
1379                         M_BNE(disp);
1380                         M_FLTMOVE(s1, REG_FTMP1);
1381                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1382                         M_CALL(REG_ITMP2);
1383                         M_INTMOVE(REG_RESULT, d);
1384                         emit_store_dst(jd, iptr, d);
1385                         break;
1386
1387                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1388
1389                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1390                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1391                         M_CVTFL(s1, d);
1392                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1393                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1394                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1395                                 ((REG_RESULT == d) ? 0 : 3);
1396                         M_BNE(disp);
1397                         M_FLTMOVE(s1, REG_FTMP1);
1398                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1399                         M_CALL(REG_ITMP2);
1400                         M_INTMOVE(REG_RESULT, d);
1401                         emit_store_dst(jd, iptr, d);
1402                         break;
1403
1404                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1405
1406                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1407                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1408                         M_CVTDL(s1, d);
1409                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1410                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1411                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1412                                 ((REG_RESULT == d) ? 0 : 3);
1413                         M_BNE(disp);
1414                         M_FLTMOVE(s1, REG_FTMP1);
1415                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1416                         M_CALL(REG_ITMP2);
1417                         M_INTMOVE(REG_RESULT, d);
1418                         emit_store_dst(jd, iptr, d);
1419                         break;
1420
1421                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1422
1423                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1424                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1425                         M_CVTFD(s1, d);
1426                         emit_store_dst(jd, iptr, d);
1427                         break;
1428
1429                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1430
1431                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1432                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1433                         M_CVTDF(s1, d);
1434                         emit_store_dst(jd, iptr, d);
1435                         break;
1436
1437                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1438                                           /* == => 0, < => 1, > => -1 */
1439
1440                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1441                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1442                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1443                         M_CLR(d);
1444                         M_MOV_IMM(1, REG_ITMP1);
1445                         M_MOV_IMM(-1, REG_ITMP2);
1446                         emit_ucomiss_reg_reg(cd, s1, s2);
1447                         M_CMOVULT(REG_ITMP1, d);
1448                         M_CMOVUGT(REG_ITMP2, d);
1449                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1450                         emit_store_dst(jd, iptr, d);
1451                         break;
1452
1453                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1454                                           /* == => 0, < => 1, > => -1 */
1455
1456                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1457                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1458                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1459                         M_CLR(d);
1460                         M_MOV_IMM(1, REG_ITMP1);
1461                         M_MOV_IMM(-1, REG_ITMP2);
1462                         emit_ucomiss_reg_reg(cd, s1, s2);
1463                         M_CMOVULT(REG_ITMP1, d);
1464                         M_CMOVUGT(REG_ITMP2, d);
1465                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1466                         emit_store_dst(jd, iptr, d);
1467                         break;
1468
1469                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1470                                           /* == => 0, < => 1, > => -1 */
1471
1472                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1473                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1474                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1475                         M_CLR(d);
1476                         M_MOV_IMM(1, REG_ITMP1);
1477                         M_MOV_IMM(-1, REG_ITMP2);
1478                         emit_ucomisd_reg_reg(cd, s1, s2);
1479                         M_CMOVULT(REG_ITMP1, d);
1480                         M_CMOVUGT(REG_ITMP2, d);
1481                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1482                         emit_store_dst(jd, iptr, d);
1483                         break;
1484
1485                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1486                                           /* == => 0, < => 1, > => -1 */
1487
1488                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1489                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1490                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1491                         M_CLR(d);
1492                         M_MOV_IMM(1, REG_ITMP1);
1493                         M_MOV_IMM(-1, REG_ITMP2);
1494                         emit_ucomisd_reg_reg(cd, s1, s2);
1495                         M_CMOVULT(REG_ITMP1, d);
1496                         M_CMOVUGT(REG_ITMP2, d);
1497                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1498                         emit_store_dst(jd, iptr, d);
1499                         break;
1500
1501
1502                 /* memory operations **************************************************/
1503
1504                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1505
1506                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1507                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1508                         /* implicit null-pointer check */
1509                         M_ILD(d, s1, OFFSET(java_array_t, size));
1510                         emit_store_dst(jd, iptr, d);
1511                         break;
1512
1513                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1514
1515                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1516                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1517                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1518                         /* implicit null-pointer check */
1519                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1520                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1521                         emit_store_dst(jd, iptr, d);
1522                         break;
1523
1524                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1525
1526                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1527                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1528                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1529                         /* implicit null-pointer check */
1530                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1531                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1532                         emit_store_dst(jd, iptr, d);
1533                         break;
1534
1535                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1536
1537                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1538                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1539                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1540                         /* implicit null-pointer check */
1541                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1542                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1543                         emit_store_dst(jd, iptr, d);
1544                         break;
1545
1546                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1547
1548                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1549                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1550                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1551                         /* implicit null-pointer check */
1552                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1553                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1554                         emit_store_dst(jd, iptr, d);
1555                         break;
1556
1557                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1558
1559                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1560                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1561                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1562                         /* implicit null-pointer check */
1563                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1564                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1565                         emit_store_dst(jd, iptr, d);
1566                         break;
1567
1568                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1569
1570                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1571                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1572                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1573                         /* implicit null-pointer check */
1574                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1575                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1576                         emit_store_dst(jd, iptr, d);
1577                         break;
1578
1579                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1580
1581                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1582                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1583                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1584                         /* implicit null-pointer check */
1585                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1586                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1587                         emit_store_dst(jd, iptr, d);
1588                         break;
1589
1590                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1591
1592                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1593                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1594                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1595                         /* implicit null-pointer check */
1596                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1597                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1598                         emit_store_dst(jd, iptr, d);
1599                         break;
1600
1601
1602                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1603
1604                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1605                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1606                         /* implicit null-pointer check */
1607                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1608                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1609                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1610                         break;
1611
1612                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1613
1614                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1615                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1616                         /* implicit null-pointer check */
1617                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1618                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1619                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1620                         break;
1621
1622                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1623
1624                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1625                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1626                         /* implicit null-pointer check */
1627                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1628                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1629                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1630                         break;
1631
1632                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1633
1634                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1635                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1636                         /* implicit null-pointer check */
1637                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1638                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1639                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1640                         break;
1641
1642                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1643
1644                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1645                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1646                         /* implicit null-pointer check */
1647                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1648                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1649                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1650                         break;
1651
1652                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1653
1654                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1655                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1656                         /* implicit null-pointer check */
1657                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1658                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1659                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1660                         break;
1661
1662                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1663
1664                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1665                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1666                         /* implicit null-pointer check */
1667                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1668                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1669                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1670                         break;
1671
1672                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1673
1674                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1675                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1676                         /* implicit null-pointer check */
1677                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1678                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1679
1680                         M_MOV(s1, REG_A0);
1681                         M_MOV(s3, REG_A1);
1682                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
1683                         M_CALL(REG_ITMP1);
1684                         emit_arraystore_check(cd, iptr);
1685
1686                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1687                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1688                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1689                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1690                         break;
1691
1692
1693                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1694
1695                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1696                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1697                         /* implicit null-pointer check */
1698                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1699                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1700                         break;
1701
1702                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1703
1704                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1705                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1706                         /* implicit null-pointer check */
1707                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1708                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1709                         break;
1710
1711                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1712
1713                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1714                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1715                         /* implicit null-pointer check */
1716                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1717                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1718                         break;
1719
1720                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1721
1722                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1723                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1724                         /* implicit null-pointer check */
1725                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1726                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1727                         break;
1728
1729                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1730
1731                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1732                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1733                         /* implicit null-pointer check */
1734                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1735
1736                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1737                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1738                         }
1739                         else {
1740                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1741                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1742                         }
1743                         break;
1744
1745                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1746
1747                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1748                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1749                         /* implicit null-pointer check */
1750                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1751                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1752                         break;
1753
1754
1755                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1756
1757                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1758                                 uf        = iptr->sx.s23.s3.uf;
1759                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1760                                 disp      = dseg_add_unique_address(cd, uf);
1761
1762 /*                              PROFILE_CYCLE_STOP; */
1763
1764                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1765
1766 /*                              PROFILE_CYCLE_START; */
1767                         }
1768                         else {
1769                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1770                                 fieldtype = fi->type;
1771                                 disp      = dseg_add_address(cd, fi->value);
1772
1773                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1774                                         PROFILE_CYCLE_STOP;
1775
1776                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1777                                                                                   fi->class, 0);
1778
1779                                         PROFILE_CYCLE_START;
1780                                 }
1781                         }
1782
1783                         /* This approach is much faster than moving the field
1784                            address inline into a register. */
1785
1786                         M_ALD(REG_ITMP1, RIP, disp);
1787
1788                         switch (fieldtype) {
1789                         case TYPE_INT:
1790                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1791                                 M_ILD(d, REG_ITMP1, 0);
1792                                 break;
1793                         case TYPE_LNG:
1794                         case TYPE_ADR:
1795                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1796                                 M_LLD(d, REG_ITMP1, 0);
1797                                 break;
1798                         case TYPE_FLT:
1799                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1800                                 M_FLD(d, REG_ITMP1, 0);
1801                                 break;
1802                         case TYPE_DBL:
1803                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1804                                 M_DLD(d, REG_ITMP1, 0);
1805                                 break;
1806                         }
1807                         emit_store_dst(jd, iptr, d);
1808                         break;
1809
1810                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1811
1812                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1813                                 uf        = iptr->sx.s23.s3.uf;
1814                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1815                                 disp      = dseg_add_unique_address(cd, uf);
1816
1817 /*                              PROFILE_CYCLE_STOP; */
1818
1819                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1820
1821 /*                              PROFILE_CYCLE_START; */
1822                         }
1823                         else {
1824                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1825                                 fieldtype = fi->type;
1826                                 disp      = dseg_add_address(cd, fi->value);
1827
1828                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1829                                         PROFILE_CYCLE_STOP;
1830
1831                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1832                                                                                   fi->class, 0);
1833
1834                                         PROFILE_CYCLE_START;
1835                                 }
1836                         }
1837
1838                         /* This approach is much faster than moving the field
1839                            address inline into a register. */
1840
1841                         M_ALD(REG_ITMP1, RIP, disp);
1842
1843                         switch (fieldtype) {
1844                         case TYPE_INT:
1845                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1846                                 M_IST(s1, REG_ITMP1, 0);
1847                                 break;
1848                         case TYPE_LNG:
1849                         case TYPE_ADR:
1850                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1851                                 M_LST(s1, REG_ITMP1, 0);
1852                                 break;
1853                         case TYPE_FLT:
1854                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1855                                 M_FST(s1, REG_ITMP1, 0);
1856                                 break;
1857                         case TYPE_DBL:
1858                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1859                                 M_DST(s1, REG_ITMP1, 0);
1860                                 break;
1861                         }
1862                         break;
1863
1864                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1865                                           /* val = value (in current instruction)     */
1866                                           /* following NOP)                           */
1867
1868                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1869                                 uf        = iptr->sx.s23.s3.uf;
1870                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1871                                 disp      = dseg_add_unique_address(cd, uf);
1872
1873 /*                              PROFILE_CYCLE_STOP; */
1874
1875                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1876
1877 /*                              PROFILE_CYCLE_START; */
1878                         }
1879                         else {
1880                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1881                                 fieldtype = fi->type;
1882                                 disp      = dseg_add_address(cd, fi->value);
1883
1884                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1885                                         PROFILE_CYCLE_STOP;
1886
1887                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1888                                                                                   fi->class, 0);
1889
1890                                         PROFILE_CYCLE_START;
1891                                 }
1892                         }
1893
1894                         /* This approach is much faster than moving the field
1895                            address inline into a register. */
1896
1897                         M_ALD(REG_ITMP1, RIP, disp);
1898
1899                         switch (fieldtype) {
1900                         case TYPE_INT:
1901                         case TYPE_FLT:
1902                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1903                                 break;
1904                         case TYPE_LNG:
1905                         case TYPE_ADR:
1906                         case TYPE_DBL:
1907                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1908                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1909                                 else {
1910                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1911                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1912                                 }
1913                                 break;
1914                         }
1915                         break;
1916
1917                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1918
1919                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1920
1921                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1922                                 uf        = iptr->sx.s23.s3.uf;
1923                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1924                                 disp      = 0;
1925
1926 /*                              PROFILE_CYCLE_STOP; */
1927
1928                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1929
1930 /*                              PROFILE_CYCLE_START; */
1931                         }
1932                         else {
1933                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1934                                 fieldtype = fi->type;
1935                                 disp      = fi->offset;
1936                         }
1937
1938                         /* implicit null-pointer check */
1939                         switch (fieldtype) {
1940                         case TYPE_INT:
1941                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1942                                 M_ILD32(d, s1, disp);
1943                                 break;
1944                         case TYPE_LNG:
1945                         case TYPE_ADR:
1946                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1947                                 M_LLD32(d, s1, disp);
1948                                 break;
1949                         case TYPE_FLT:
1950                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1951                                 M_FLD32(d, s1, disp);
1952                                 break;
1953                         case TYPE_DBL:
1954                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1955                                 M_DLD32(d, s1, disp);
1956                                 break;
1957                         }
1958                         emit_store_dst(jd, iptr, d);
1959                         break;
1960
1961                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1962
1963                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1964                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1965
1966                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1967                                 uf        = iptr->sx.s23.s3.uf;
1968                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1969                                 disp      = 0;
1970
1971 /*                              PROFILE_CYCLE_STOP; */
1972
1973                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1974
1975 /*                              PROFILE_CYCLE_START; */
1976                         }
1977                         else {
1978                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1979                                 fieldtype = fi->type;
1980                                 disp      = fi->offset;
1981                         }
1982
1983                         /* implicit null-pointer check */
1984                         switch (fieldtype) {
1985                         case TYPE_INT:
1986                                 M_IST32(s2, s1, disp);
1987                                 break;
1988                         case TYPE_LNG:
1989                         case TYPE_ADR:
1990                                 M_LST32(s2, s1, disp);
1991                                 break;
1992                         case TYPE_FLT:
1993                                 M_FST32(s2, s1, disp);
1994                                 break;
1995                         case TYPE_DBL:
1996                                 M_DST32(s2, s1, disp);
1997                                 break;
1998                         }
1999                         break;
2000
2001                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2002                                           /* val = value (in current instruction)     */
2003                                           /* following NOP)                           */
2004
2005                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2006
2007                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2008                                 uf        = iptr->sx.s23.s3.uf;
2009                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2010                                 disp      = 0;
2011
2012 /*                              PROFILE_CYCLE_STOP; */
2013
2014                                 patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
2015
2016 /*                              PROFILE_CYCLE_START; */
2017                         }
2018                         else {
2019                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2020                                 fieldtype = fi->type;
2021                                 disp      = fi->offset;
2022                         }
2023
2024                         /* implicit null-pointer check */
2025                         switch (fieldtype) {
2026                         case TYPE_INT:
2027                         case TYPE_FLT:
2028                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2029                                 break;
2030                         case TYPE_LNG:
2031                         case TYPE_ADR:
2032                         case TYPE_DBL:
2033                                 /* XXX why no check for IS_IMM32? */
2034                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2035                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2036                                 break;
2037                         }
2038                         break;
2039
2040
2041                 /* branch operations **************************************************/
2042
2043                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2044
2045                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2046                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2047
2048                         PROFILE_CYCLE_STOP;
2049
2050 #ifdef ENABLE_VERIFIER
2051                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2052                                 uc = iptr->sx.s23.s2.uc;
2053
2054                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2055                         }
2056 #endif /* ENABLE_VERIFIER */
2057
2058                         M_CALL_IMM(0);                            /* passing exception pc */
2059                         M_POP(REG_ITMP2_XPC);
2060
2061                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2062                         M_JMP(REG_ITMP3);
2063                         break;
2064
2065                 case ICMD_GOTO:         /* ... ==> ...                                */
2066                 case ICMD_RET:
2067
2068                         emit_br(cd, iptr->dst.block);
2069                         ALIGNCODENOP;
2070                         break;
2071
2072                 case ICMD_JSR:          /* ... ==> ...                                */
2073
2074                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2075                         ALIGNCODENOP;
2076                         break;
2077
2078                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2079                 case ICMD_IFNONNULL:
2080
2081                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2082                         M_TEST(s1);
2083                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2084                         break;
2085
2086                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2087                 case ICMD_IFLT:
2088                 case ICMD_IFLE:
2089                 case ICMD_IFNE:
2090                 case ICMD_IFGT:
2091                 case ICMD_IFGE:
2092
2093                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2094                         M_ICMP_IMM(iptr->sx.val.i, s1);
2095                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2096                         break;
2097
2098                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2099                 case ICMD_IF_LNE:
2100                 case ICMD_IF_LLT:
2101                 case ICMD_IF_LGE:
2102                 case ICMD_IF_LGT:
2103                 case ICMD_IF_LLE:
2104
2105                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2106                         if (IS_IMM32(iptr->sx.val.l))
2107                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2108                         else {
2109                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2110                                 M_LCMP(REG_ITMP2, s1);
2111                         }
2112                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2113                         break;
2114
2115                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2116                 case ICMD_IF_ICMPNE:
2117                 case ICMD_IF_ICMPLT:
2118                 case ICMD_IF_ICMPGE:
2119                 case ICMD_IF_ICMPGT:
2120                 case ICMD_IF_ICMPLE:
2121
2122                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2123                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2124                         M_ICMP(s2, s1);
2125                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2126                         break;
2127
2128                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2129                 case ICMD_IF_ACMPNE:
2130
2131                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2132                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2133                         M_LCMP(s2, s1);
2134                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2135                         break;
2136
2137                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2138                 case ICMD_IF_LCMPNE:
2139                 case ICMD_IF_LCMPLT:
2140                 case ICMD_IF_LCMPGE:
2141                 case ICMD_IF_LCMPGT:
2142                 case ICMD_IF_LCMPLE:
2143
2144                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2145                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2146                         M_LCMP(s2, s1);
2147                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2148                         break;
2149
2150                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2151                 case ICMD_LRETURN:
2152
2153                         REPLACEMENT_POINT_RETURN(cd, iptr);
2154                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2155                         M_INTMOVE(s1, REG_RESULT);
2156                         goto nowperformreturn;
2157
2158                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2159
2160                         REPLACEMENT_POINT_RETURN(cd, iptr);
2161                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2162                         M_INTMOVE(s1, REG_RESULT);
2163
2164 #ifdef ENABLE_VERIFIER
2165                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2166                                 uc = iptr->sx.s23.s2.uc;
2167
2168                                 PROFILE_CYCLE_STOP;
2169
2170                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2171
2172                                 PROFILE_CYCLE_START;
2173                         }
2174 #endif /* ENABLE_VERIFIER */
2175                         goto nowperformreturn;
2176
2177                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2178                 case ICMD_DRETURN:
2179
2180                         REPLACEMENT_POINT_RETURN(cd, iptr);
2181                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2182                         M_FLTMOVE(s1, REG_FRESULT);
2183                         goto nowperformreturn;
2184
2185                 case ICMD_RETURN:      /* ...  ==> ...                                */
2186
2187                         REPLACEMENT_POINT_RETURN(cd, iptr);
2188
2189 nowperformreturn:
2190                         {
2191                         s4 i, p;
2192
2193                         p = cd->stackframesize;
2194
2195 #if !defined(NDEBUG)
2196                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2197                                 emit_verbosecall_exit(jd);
2198 #endif /* !defined(NDEBUG) */
2199
2200 #if defined(ENABLE_THREADS)
2201                         if (checksync && code_is_synchronized(code)) {
2202                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2203
2204                                 /* we need to save the proper return value */
2205                                 switch (iptr->opc) {
2206                                 case ICMD_IRETURN:
2207                                 case ICMD_ARETURN:
2208                                 case ICMD_LRETURN:
2209                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2210                                         break;
2211                                 case ICMD_FRETURN:
2212                                 case ICMD_DRETURN:
2213                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2214                                         break;
2215                                 }
2216
2217                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2218                                 M_CALL(REG_ITMP1);
2219
2220                                 /* and now restore the proper return value */
2221                                 switch (iptr->opc) {
2222                                 case ICMD_IRETURN:
2223                                 case ICMD_ARETURN:
2224                                 case ICMD_LRETURN:
2225                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2226                                         break;
2227                                 case ICMD_FRETURN:
2228                                 case ICMD_DRETURN:
2229                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2230                                         break;
2231                                 }
2232                         }
2233 #endif
2234
2235                         /* restore saved registers */
2236
2237                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2238                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2239                         }
2240                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2241                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2242                         }
2243
2244                         /* deallocate stack */
2245
2246                         if (cd->stackframesize)
2247                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2248
2249                         /* generate method profiling code */
2250
2251                         PROFILE_CYCLE_STOP;
2252
2253                         M_RET;
2254                         }
2255                         break;
2256
2257
2258                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2259                         {
2260                                 s4 i, l;
2261                                 branch_target_t *table;
2262
2263                                 table = iptr->dst.table;
2264
2265                                 l = iptr->sx.s23.s2.tablelow;
2266                                 i = iptr->sx.s23.s3.tablehigh;
2267
2268                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2269                                 M_INTMOVE(s1, REG_ITMP1);
2270
2271                                 if (l != 0)
2272                                         M_ISUB_IMM(l, REG_ITMP1);
2273
2274                                 /* number of targets */
2275                                 i = i - l + 1;
2276
2277                 /* range check */
2278
2279                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2280                                 emit_bugt(cd, table[0].block);
2281
2282                                 /* build jump table top down and use address of lowest entry */
2283
2284                                 table += i;
2285
2286                                 while (--i >= 0) {
2287                                         dseg_add_target(cd, table->block);
2288                                         --table;
2289                                 }
2290
2291                                 /* length of dataseg after last dseg_add_target is used
2292                                    by load */
2293
2294                                 M_MOV_IMM(0, REG_ITMP2);
2295                                 dseg_adddata(cd);
2296                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2297                                 M_JMP(REG_ITMP1);
2298                         }
2299                         break;
2300
2301
2302                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2303                         {
2304                                 s4 i;
2305                                 lookup_target_t *lookup;
2306
2307                                 lookup = iptr->dst.lookup;
2308
2309                                 i = iptr->sx.s23.s2.lookupcount;
2310
2311                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2312                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2313
2314                                 while (--i >= 0) {
2315                                         M_ICMP_IMM(lookup->value, s1);
2316                                         emit_beq(cd, lookup->target.block);
2317                                         lookup++;
2318                                 }
2319
2320                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2321                                 ALIGNCODENOP;
2322                         }
2323                         break;
2324
2325
2326                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2327
2328                         REPLACEMENT_POINT_FORGC_BUILTIN(cd, iptr);
2329
2330                         bte = iptr->sx.s23.s3.bte;
2331                         md  = bte->md;
2332                         goto gen_method;
2333
2334                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2335
2336                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2337                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2338                 case ICMD_INVOKEINTERFACE:
2339
2340                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2341
2342                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2343                                 lm = NULL;
2344                                 um = iptr->sx.s23.s3.um;
2345                                 md = um->methodref->parseddesc.md;
2346                         }
2347                         else {
2348                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2349                                 um = NULL;
2350                                 md = lm->parseddesc;
2351                         }
2352
2353 gen_method:
2354                         s3 = md->paramcount;
2355
2356                         MCODECHECK((20 * s3) + 128);
2357
2358                         /* copy arguments to registers or stack location */
2359
2360                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2361                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2362                                 d   = md->params[s3].regoff;
2363
2364                                 /* already preallocated (ARGVAR)? */
2365
2366                                 if (var->flags & PREALLOC)
2367                                         continue;
2368
2369                                 if (IS_INT_LNG_TYPE(var->type)) {
2370                                         if (!md->params[s3].inmemory) {
2371                                                 s1 = emit_load(jd, iptr, var, d);
2372                                                 M_INTMOVE(s1, d);
2373                                         }
2374                                         else {
2375                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2376                                                 M_LST(s1, REG_SP, d);
2377                                         }
2378                                 }
2379                                 else {
2380                                         if (!md->params[s3].inmemory) {
2381                                                 s1 = emit_load(jd, iptr, var, d);
2382                                                 M_FLTMOVE(s1, d);
2383                                         }
2384                                         else {
2385                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2386
2387                                                 if (IS_2_WORD_TYPE(var->type))
2388                                                         M_DST(s1, REG_SP, d);
2389                                                 else
2390                                                         M_FST(s1, REG_SP, d);
2391                                         }
2392                                 }
2393                         }
2394
2395                         /* generate method profiling code */
2396
2397                         PROFILE_CYCLE_STOP;
2398
2399                         switch (iptr->opc) {
2400                         case ICMD_BUILTIN:
2401                                 if (bte->stub == NULL) {
2402                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2403                                 }
2404                                 else {
2405                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2406                                 }
2407                                 M_CALL(REG_ITMP1);
2408                                 break;
2409
2410                         case ICMD_INVOKESPECIAL:
2411                                 emit_nullpointer_check(cd, iptr, REG_A0);
2412                                 /* fall through */
2413
2414                         case ICMD_INVOKESTATIC:
2415                                 if (lm == NULL) {
2416                                         disp = dseg_add_unique_address(cd, um);
2417
2418                                         patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
2419                                                                                   um, disp);
2420                                 }
2421                                 else {
2422                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2423                                 }
2424
2425                                 M_ALD(REG_ITMP2, RIP, disp);
2426                                 M_CALL(REG_ITMP2);
2427                                 break;
2428
2429                         case ICMD_INVOKEVIRTUAL:
2430                                 if (lm == NULL) {
2431                                         patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
2432
2433                                         s1 = 0;
2434                                 }
2435                                 else {
2436                                         s1 = OFFSET(vftbl_t, table[0]) +
2437                                                 sizeof(methodptr) * lm->vftblindex;
2438                                 }
2439
2440                                 /* implicit null-pointer check */
2441                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2442                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2443                                 M_CALL(REG_ITMP3);
2444                                 break;
2445
2446                         case ICMD_INVOKEINTERFACE:
2447                                 if (lm == NULL) {
2448                                         patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
2449
2450                                         s1 = 0;
2451                                         s2 = 0;
2452                                 }
2453                                 else {
2454                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2455                                                 sizeof(methodptr) * lm->class->index;
2456
2457                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2458                                 }
2459
2460                                 /* implicit null-pointer check */
2461                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2462                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2463                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2464                                 M_CALL(REG_ITMP3);
2465                                 break;
2466                         }
2467
2468                         /* generate method profiling code */
2469
2470                         PROFILE_CYCLE_START;
2471
2472                         /* store size of call code in replacement point */
2473
2474                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2475                         REPLACEMENT_POINT_FORGC_BUILTIN_RETURN(cd, iptr);
2476
2477                         /* store return value */
2478
2479                         switch (md->returntype.type) {
2480                         case TYPE_INT:
2481                         case TYPE_LNG:
2482                         case TYPE_ADR:
2483                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2484                                 M_INTMOVE(REG_RESULT, s1);
2485                                 emit_store_dst(jd, iptr, s1);
2486                                 break;
2487                         case TYPE_FLT:
2488                         case TYPE_DBL:
2489                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2490                                 M_FLTMOVE(REG_FRESULT, s1);
2491                                 emit_store_dst(jd, iptr, s1);
2492                                 break;
2493                         default:
2494                                 /* TYPE_VOID */
2495                                 break;
2496                         }
2497                         break;
2498
2499
2500                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2501
2502                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2503                                 /* object type cast-check */
2504
2505                                 classinfo *super;
2506                                 s4         superindex;
2507
2508                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2509                                         super      = NULL;
2510                                         superindex = 0;
2511                                 }
2512                                 else {
2513                                         super      = iptr->sx.s23.s3.c.cls;
2514                                         superindex = super->index;
2515                                 }
2516
2517                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2518                                         CODEGEN_CRITICAL_SECTION_NEW;
2519
2520                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2521
2522                                 /* if class is not resolved, check which code to call */
2523
2524                                 if (super == NULL) {
2525                                         M_TEST(s1);
2526                                         emit_label_beq(cd, BRANCH_LABEL_1);
2527
2528                                         patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2529                                                                                   iptr->sx.s23.s3.c.ref, 0);
2530
2531                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2532                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2533                                         emit_label_beq(cd, BRANCH_LABEL_2);
2534                                 }
2535
2536                                 /* interface checkcast code */
2537
2538                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2539                                         if (super != NULL) {
2540                                                 M_TEST(s1);
2541                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2542                                         }
2543
2544                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2545
2546                                         if (super == NULL) {
2547                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
2548                                                                                           iptr->sx.s23.s3.c.ref,
2549                                                                                           0);
2550                                         }
2551
2552                                         M_ILD32(REG_ITMP3,
2553                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2554                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2555                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2556
2557                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2558                                                         OFFSET(vftbl_t, interfacetable[0]) -
2559                                                         superindex * sizeof(methodptr*));
2560                                         M_TEST(REG_ITMP3);
2561                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2562
2563                                         if (super == NULL)
2564                                                 emit_label_br(cd, BRANCH_LABEL_4);
2565                                         else
2566                                                 emit_label(cd, BRANCH_LABEL_3);
2567                                 }
2568
2569                                 /* class checkcast code */
2570
2571                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2572                                         if (super == NULL) {
2573                                                 emit_label(cd, BRANCH_LABEL_2);
2574
2575                                                 cr   = iptr->sx.s23.s3.c.ref;
2576                                                 disp = dseg_add_unique_address(cd, cr);
2577
2578                                                 patcher_add_patch_ref(jd,
2579                                                                                           PATCHER_resolve_classref_to_vftbl,
2580                                                                                           cr, disp);
2581                                         }
2582                                         else {
2583                                                 M_TEST(s1);
2584                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2585
2586                                                 disp = dseg_add_address(cd, super->vftbl);
2587                                         }
2588
2589                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2590                                         M_ALD(REG_ITMP3, RIP, disp);
2591
2592                                         CODEGEN_CRITICAL_SECTION_START;
2593
2594                                         M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2595
2596                                         /*                                      if (s1 != REG_ITMP1) { */
2597                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2598                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2599                                         /*                                                                                              REG_ITMP1); */
2600                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2601                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2602                                         /*                                                                                              REG_ITMP3); */
2603                                         /*  #if defined(ENABLE_THREADS) */
2604                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2605                                         /*  #endif */
2606                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2607
2608                                         /*                                      } else { */
2609
2610                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
2611                                         M_ISUB(REG_ITMP3, REG_ITMP2);
2612                                         M_ALD(REG_ITMP3, RIP, disp);
2613                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2614                                         /*                                      } */
2615
2616                                         CODEGEN_CRITICAL_SECTION_END;
2617
2618                                         M_ICMP(REG_ITMP3, REG_ITMP2);
2619                                         emit_classcast_check(cd, iptr, BRANCH_UGT, REG_ITMP3, s1);
2620
2621                                         if (super != NULL)
2622                                                 emit_label(cd, BRANCH_LABEL_5);
2623                                 }
2624
2625                                 if (super == NULL) {
2626                                         emit_label(cd, BRANCH_LABEL_1);
2627                                         emit_label(cd, BRANCH_LABEL_4);
2628                                 }
2629
2630                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2631                         }
2632                         else {
2633                                 /* array type cast-check */
2634
2635                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2636                                 M_INTMOVE(s1, REG_A0);
2637
2638                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2639                                         cr   = iptr->sx.s23.s3.c.ref;
2640                                         disp = dseg_add_unique_address(cd, cr);
2641
2642                                         patcher_add_patch_ref(jd,
2643                                                                                   PATCHER_resolve_classref_to_classinfo,
2644                                                                                   cr, disp);
2645                                 }
2646                                 else {
2647                                         disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2648                                 }
2649
2650                                 M_ALD(REG_A1, RIP, disp);
2651                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2652                                 M_CALL(REG_ITMP1);
2653
2654                                 /* s1 may have been destroyed over the function call */
2655                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2656                                 M_TEST(REG_RESULT);
2657                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2658
2659                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2660                         }
2661
2662                         M_INTMOVE(s1, d);
2663                         emit_store_dst(jd, iptr, d);
2664                         break;
2665
2666                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2667
2668                         {
2669                         classinfo *super;
2670                         s4         superindex;
2671
2672                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2673                                 super      = NULL;
2674                                 superindex = 0;
2675                         }
2676                         else {
2677                                 super      = iptr->sx.s23.s3.c.cls;
2678                                 superindex = super->index;
2679                         }
2680
2681                         if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2682                                 CODEGEN_CRITICAL_SECTION_NEW;
2683
2684                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2685                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2686
2687                         if (s1 == d) {
2688                                 M_INTMOVE(s1, REG_ITMP1);
2689                                 s1 = REG_ITMP1;
2690                         }
2691
2692                         M_CLR(d);
2693
2694                         /* if class is not resolved, check which code to call */
2695
2696                         if (super == NULL) {
2697                                 M_TEST(s1);
2698                                 emit_label_beq(cd, BRANCH_LABEL_1);
2699
2700                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2701                                                                           iptr->sx.s23.s3.c.ref, 0);
2702
2703                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2704                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2705                                 emit_label_beq(cd, BRANCH_LABEL_2);
2706                         }
2707
2708                         /* interface instanceof code */
2709
2710                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2711                                 if (super != NULL) {
2712                                         M_TEST(s1);
2713                                         emit_label_beq(cd, BRANCH_LABEL_3);
2714                                 }
2715
2716                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2717
2718                                 if (super == NULL) {
2719                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
2720                                                                                   iptr->sx.s23.s3.c.ref, 0);
2721                                 }
2722
2723                                 M_ILD32(REG_ITMP3,
2724                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2725                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2726
2727                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2728
2729                                 M_BLE(a);
2730                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2731                                                 OFFSET(vftbl_t, interfacetable[0]) -
2732                                                 superindex * sizeof(methodptr*));
2733                                 M_TEST(REG_ITMP1);
2734                                 M_SETNE(d);
2735
2736                                 if (super == NULL)
2737                                         emit_label_br(cd, BRANCH_LABEL_4);
2738                                 else
2739                                         emit_label(cd, BRANCH_LABEL_3);
2740                         }
2741
2742                         /* class instanceof code */
2743
2744                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2745                                 if (super == NULL) {
2746                                         emit_label(cd, BRANCH_LABEL_2);
2747
2748                                         cr   = iptr->sx.s23.s3.c.ref;
2749                                         disp = dseg_add_unique_address(cd, cr);
2750
2751                                         patcher_add_patch_ref(jd,
2752                                                                                   PATCHER_resolve_classref_to_vftbl,
2753                                                                                   cr, disp);
2754                                 }
2755                                 else {
2756                                         M_TEST(s1);
2757                                         emit_label_beq(cd, BRANCH_LABEL_5);
2758
2759                                         disp = dseg_add_address(cd, super->vftbl);
2760                                 }
2761
2762                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2763                                 M_ALD(REG_ITMP2, RIP, disp);
2764
2765                                 CODEGEN_CRITICAL_SECTION_START;
2766
2767                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
2768                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2769                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2770
2771                                 CODEGEN_CRITICAL_SECTION_END;
2772
2773                                 M_ISUB(REG_ITMP2, REG_ITMP1);
2774                                 M_CLR(d); /* may be REG_ITMP2 */
2775                                 M_ICMP(REG_ITMP3, REG_ITMP1);
2776                                 M_SETULE(d);
2777
2778                                 if (super != NULL)
2779                                         emit_label(cd, BRANCH_LABEL_5);
2780                         }
2781
2782                         if (super == NULL) {
2783                                 emit_label(cd, BRANCH_LABEL_1);
2784                                 emit_label(cd, BRANCH_LABEL_4);
2785                         }
2786
2787                         emit_store_dst(jd, iptr, d);
2788                         }
2789                         break;
2790
2791                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2792
2793                         /* check for negative sizes and copy sizes to stack if necessary  */
2794
2795                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2796
2797                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2798
2799                                 /* copy SAVEDVAR sizes to stack */
2800                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2801
2802                                 /* Already Preallocated? */
2803                                 if (!(var->flags & PREALLOC)) {
2804                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2805                                         M_LST(s2, REG_SP, s1 * 8);
2806                                 }
2807                         }
2808
2809                         /* a0 = dimension count */
2810
2811                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2812
2813                         /* is a patcher function set? */
2814
2815                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2816                                 cr   = iptr->sx.s23.s3.c.ref;
2817                                 disp = dseg_add_unique_address(cd, cr);
2818
2819                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
2820                                                                           cr, disp);
2821                         }
2822                         else {
2823                                 disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2824                         }
2825
2826                         /* a1 = classinfo */
2827
2828                         M_ALD(REG_A1, RIP, disp);
2829
2830                         /* a2 = pointer to dimensions = stack pointer */
2831
2832                         M_MOV(REG_SP, REG_A2);
2833
2834                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2835                         M_CALL(REG_ITMP1);
2836
2837                         /* check for exception before result assignment */
2838
2839                         emit_exception_check(cd, iptr);
2840
2841                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2842                         M_INTMOVE(REG_RESULT, s1);
2843                         emit_store_dst(jd, iptr, s1);
2844                         break;
2845
2846                 default:
2847                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2848                                                                                    iptr->opc);
2849                         return false;
2850         } /* switch */
2851
2852         } /* for instruction */
2853
2854         MCODECHECK(512); /* XXX require a lower number? */
2855
2856         /* At the end of a basic block we may have to append some nops,
2857            because the patcher stub calling code might be longer than the
2858            actual instruction. So codepatching does not change the
2859            following block unintentionally. */
2860
2861         if (cd->mcodeptr < cd->lastmcodeptr) {
2862                 while (cd->mcodeptr < cd->lastmcodeptr) {
2863                         M_NOP;
2864                 }
2865         }
2866
2867         } /* if (bptr -> flags >= BBREACHED) */
2868         } /* for basic block */
2869
2870         dseg_createlinenumbertable(cd);
2871
2872         /* Generate patcher traps. */
2873
2874         emit_patcher_traps(jd);
2875
2876         /* everything's ok */
2877
2878         return true;
2879 }
2880
2881
2882 /* codegen_emit_stub_native ****************************************************
2883
2884    Emits a stub routine which calls a native method.
2885
2886 *******************************************************************************/
2887
2888 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f, int skipparams)
2889 {
2890         methodinfo  *m;
2891         codeinfo    *code;
2892         codegendata *cd;
2893         methoddesc  *md;
2894         int          i, j;
2895         int          s1, s2;
2896         int          disp;
2897
2898         /* Sanity check. */
2899
2900         assert(f != NULL);
2901
2902         /* Get required compiler data. */
2903
2904         m    = jd->m;
2905         code = jd->code;
2906         cd   = jd->cd;
2907
2908         /* initialize variables */
2909
2910         md = m->parseddesc;
2911
2912         /* calculate stack frame size */
2913
2914         cd->stackframesize =
2915                 sizeof(stackframeinfo_t) / SIZEOF_VOID_P +
2916                 sizeof(localref_table) / SIZEOF_VOID_P +
2917                 md->paramcount +
2918                 (md->returntype.type == TYPE_VOID ? 0 : 1) +
2919                 nmd->memuse;
2920
2921         ALIGN_ODD(cd->stackframesize);              /* keep stack 16-byte aligned */
2922
2923         /* create method header */
2924
2925         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2926         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2927         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
2928         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2929         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2930         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2931         (void) dseg_addlinenumbertablesize(cd);
2932         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
2933
2934 #if defined(ENABLE_PROFILING)
2935         /* generate native method profiling code */
2936
2937         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
2938                 /* count frequency */
2939
2940                 M_MOV_IMM(code, REG_ITMP3);
2941                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
2942         }
2943 #endif
2944
2945         /* generate stub code */
2946
2947         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2948
2949 #if defined(ENABLE_GC_CACAO)
2950         /* Save callee saved integer registers in stackframeinfo (GC may
2951            need to recover them during a collection). */
2952
2953         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
2954                 OFFSET(stackframeinfo_t, intregs);
2955
2956         for (i = 0; i < INT_SAV_CNT; i++)
2957                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2958 #endif
2959
2960         /* save integer and float argument registers */
2961
2962         for (i = 0; i < md->paramcount; i++) {
2963                 if (!md->params[i].inmemory) {
2964                         s1 = md->params[i].regoff;
2965
2966                         switch (md->paramtypes[i].type) {
2967                         case TYPE_INT:
2968                         case TYPE_LNG:
2969                         case TYPE_ADR:
2970                                 M_LST(s1, REG_SP, i * 8);
2971                                 break;
2972                         case TYPE_FLT:
2973                         case TYPE_DBL:
2974                                 M_DST(s1, REG_SP, i * 8);
2975                                 break;
2976                         }
2977                 }
2978         }
2979
2980         /* create dynamic stack info */
2981
2982         M_MOV(REG_SP, REG_A0);
2983         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
2984         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
2985         M_CALL(REG_ITMP1);
2986
2987         /* remember class argument */
2988
2989         if (m->flags & ACC_STATIC)
2990                 M_MOV(REG_RESULT, REG_ITMP2);
2991
2992         /* restore integer and float argument registers */
2993
2994         for (i = 0; i < md->paramcount; i++) {
2995                 if (!md->params[i].inmemory) {
2996                         s1 = md->params[i].regoff;
2997
2998                         switch (md->paramtypes[i].type) {
2999                         case TYPE_INT:
3000                         case TYPE_LNG:
3001                         case TYPE_ADR:
3002                                 M_LLD(s1, REG_SP, i * 8);
3003                                 break;
3004                         case TYPE_FLT:
3005                         case TYPE_DBL:
3006                                 M_DLD(s1, REG_SP, i * 8);
3007                                 break;
3008                         }
3009                 }
3010         }
3011
3012         /* Copy or spill arguments to new locations. */
3013
3014         for (i = md->paramcount - 1, j = i + skipparams; i >= 0; i--, j--) {
3015                 s2 = nmd->params[j].regoff;
3016
3017                 switch (md->paramtypes[i].type) {
3018                 case TYPE_INT:
3019                 case TYPE_LNG:
3020                 case TYPE_ADR:
3021                         if (!md->params[i].inmemory) {
3022                                 s1 = md->params[i].regoff;
3023
3024                                 if (!nmd->params[j].inmemory)
3025                                         M_INTMOVE(s1, s2);
3026                                 else
3027                                         M_LST(s1, REG_SP, s2);
3028                         }
3029                         else {
3030                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3031                                 M_LLD(REG_ITMP1, REG_SP, s1);
3032                                 M_LST(REG_ITMP1, REG_SP, s2);
3033                         }
3034                         break;
3035                 case TYPE_FLT:
3036                         /* We only copy spilled float arguments, as the float
3037                            argument registers keep unchanged. */
3038
3039                         if (md->params[i].inmemory) {
3040                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3041
3042                                 M_FLD(REG_FTMP1, REG_SP, s1);
3043                                 M_FST(REG_FTMP1, REG_SP, s2);
3044                         }
3045                         break;
3046                 case TYPE_DBL:
3047                         if (md->params[i].inmemory) {
3048                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3049                                 M_DLD(REG_FTMP1, REG_SP, s1);
3050                                 M_DST(REG_FTMP1, REG_SP, s2);
3051                         }
3052                         break;
3053                 }
3054         }
3055
3056         /* Handle native Java methods. */
3057
3058         if (m->flags & ACC_NATIVE) {
3059                 /* put class into second argument register */
3060
3061                 if (m->flags & ACC_STATIC)
3062                         M_MOV(REG_ITMP2, REG_A1);
3063
3064                 /* put env into first argument register */
3065
3066                 M_MOV_IMM(_Jv_env, REG_A0);
3067         }
3068
3069         /* Call the native function. */
3070
3071         disp = dseg_add_functionptr(cd, f);
3072         M_ALD(REG_ITMP1, RIP, disp);
3073         M_CALL(REG_ITMP1);
3074
3075         /* save return value */
3076
3077         switch (md->returntype.type) {
3078         case TYPE_INT:
3079         case TYPE_LNG:
3080         case TYPE_ADR:
3081                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3082                 break;
3083         case TYPE_FLT:
3084         case TYPE_DBL:
3085                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3086                 break;
3087         case TYPE_VOID:
3088                 break;
3089         }
3090
3091         /* remove native stackframe info */
3092
3093         M_MOV(REG_SP, REG_A0);
3094         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3095         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3096         M_CALL(REG_ITMP1);
3097         M_MOV(REG_RESULT, REG_ITMP3);
3098
3099         /* restore return value */
3100
3101         switch (md->returntype.type) {
3102         case TYPE_INT:
3103         case TYPE_LNG:
3104         case TYPE_ADR:
3105                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3106                 break;
3107         case TYPE_FLT:
3108         case TYPE_DBL:
3109                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3110                 break;
3111         case TYPE_VOID:
3112                 break;
3113         }
3114
3115 #if defined(ENABLE_GC_CACAO)
3116         /* Restore callee saved integer registers from stackframeinfo (GC
3117            might have modified them during a collection). */
3118
3119         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
3120                 OFFSET(stackframeinfo_t, intregs);
3121
3122         for (i = 0; i < INT_SAV_CNT; i++)
3123                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3124 #endif
3125
3126         /* remove stackframe */
3127
3128         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3129
3130         /* test for exception */
3131
3132         M_TEST(REG_ITMP3);
3133         M_BNE(1);
3134         M_RET;
3135
3136         /* handle exception */
3137
3138         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3139         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3140         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3141
3142         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3143         M_JMP(REG_ITMP3);
3144 }
3145
3146
3147 /*
3148  * These are local overrides for various environment variables in Emacs.
3149  * Please do not remove this and leave it at the end of the file, where
3150  * Emacs will automagically detect them.
3151  * ---------------------------------------------------------------------
3152  * Local variables:
3153  * mode: c
3154  * indent-tabs-mode: t
3155  * c-basic-offset: 4
3156  * tab-width: 4
3157  * End:
3158  * vim:noexpandtab:sw=4:ts=4:
3159  */