src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25 */
  26
  27
  28 #include "config.h"
  29
  30 #include <assert.h>
  31 #include <stdio.h>
  32 #include <stdint.h>
  33
  34 #include "vm/types.h"
  35
  36 #include "md-abi.h"
  37
  38 #include "vm/jit/x86_64/arch.h"
  39 #include "vm/jit/x86_64/codegen.h"
  40 #include "vm/jit/x86_64/emit.h"
  41
  42 #include "mm/memory.h"
  43
  44 #include "native/jni.h"
  45 #include "native/localref.h"
  46 #include "native/native.h"
  47
  48 #include "threads/lock-common.h"
  49
  50 #include "vm/builtin.h"
  51 #include "vm/exceptions.h"
  52 #include "vm/global.h"
  53 #include "vm/stringlocal.h"
  54 #include "vm/vm.h"
  55
  56 #include "vm/jit/abi.h"
  57 #include "vm/jit/asmpart.h"
  58 #include "vm/jit/code.h"
  59 #include "vm/jit/codegen-common.h"
  60 #include "vm/jit/dseg.h"
  61 #include "vm/jit/emit-common.h"
  62 #include "vm/jit/jit.h"
  63 #include "vm/jit/methodheader.h"
  64 #include "vm/jit/parse.h"
  65 #include "vm/jit/patcher-common.h"
  66 #include "vm/jit/reg.h"
  67 #include "vm/jit/replace.h"
  68 #include "vm/jit/stacktrace.h"
  69
  70 #if defined(ENABLE_LSRA)
  71 # include "vm/jit/allocator/lsra.h"
  72 #endif
  73
  74 #include "vmcore/loader.h"
  75 #include "vmcore/options.h"
  76 #include "vmcore/statistics.h"
  77
  78
  79 /* codegen_emit ****************************************************************
  80
  81    Generates machine code.
  82
  83 *******************************************************************************/
  84
  85 bool codegen_emit(jitdata *jd)
  86 {
  87         methodinfo         *m;
  88         codeinfo           *code;
  89         codegendata        *cd;
  90         registerdata       *rd;
  91         s4                  len, s1, s2, s3, d, disp;
  92         u2                  currentline;
  93         ptrint              a;
  94         varinfo            *var, *dst;
  95         basicblock         *bptr;
  96         instruction        *iptr;
  97         exception_entry    *ex;
  98         constant_classref  *cr;
  99         unresolved_class   *uc;
 100         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 101         unresolved_method  *um;
 102         builtintable_entry *bte;
 103         methoddesc         *md;
 104         fieldinfo          *fi;
 105         unresolved_field   *uf;
 106         s4                  fieldtype;
 107         s4                 varindex;
 108
 109         /* get required compiler data */
 110
 111         m    = jd->m;
 112         code = jd->code;
 113         cd   = jd->cd;
 114         rd   = jd->rd;
 115
 116         /* prevent compiler warnings */
 117
 118         d   = 0;
 119         lm  = NULL;
 120         um  = NULL;
 121         bte = NULL;
 122
 123         {
 124         s4 i, p, t, l;
 125         s4 savedregs_num;
 126
 127         savedregs_num = 0;
 128
 129         /* space to save used callee saved registers */
 130
 131         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 132         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 133
 134         cd->stackframesize = rd->memuse + savedregs_num;
 135
 136 #if defined(ENABLE_THREADS)
 137         /* space to save argument of monitor_enter */
 138
 139         if (checksync && code_is_synchronized(code))
 140                 cd->stackframesize++;
 141 #endif
 142
 143         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 144            native code e.g. libc or jni (alignment problems with
 145            movaps). */
 146
 147         if (!code_is_leafmethod(code) || opt_verbosecall)
 148                 cd->stackframesize |= 0x1;
 149
 150         /* create method header */
 151
 152         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 153         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 154
 155         /* IsSync contains the offset relative to the stack pointer for the
 156            argument of monitor_exit used in the exception handler. Since the
 157            offset could be zero and give a wrong meaning of the flag it is
 158            offset by one.
 159         */
 160         /* XXX Remove this "offset by one". */
 161
 162         code->synchronizedoffset = (rd->memuse + 1) * 8;
 163
 164         /* REMOVEME dummy IsSync */
 165         (void) dseg_add_unique_s4(cd, 0);
 166
 167         if (code_is_leafmethod(code))
 168                 (void) dseg_add_unique_s4(cd, 1);                  /* IsLeaf          */
 169         else
 170                 (void) dseg_add_unique_s4(cd, 0);                  /* IsLeaf          */
 171
 172         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 173         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 174
 175         (void) dseg_addlinenumbertablesize(cd);
 176
 177         (void) dseg_add_unique_s4(cd, jd->exceptiontablelength); /* ExTableSize   */
 178
 179         /* create exception table */
 180
 181         for (ex = jd->exceptiontable; ex != NULL; ex = ex->down) {
 182                 dseg_add_target(cd, ex->start);
 183                 dseg_add_target(cd, ex->end);
 184                 dseg_add_target(cd, ex->handler);
 185                 (void) dseg_add_unique_address(cd, ex->catchtype.any);
 186         }
 187
 188 #if defined(ENABLE_PROFILING)
 189         /* generate method profiling code */
 190
 191         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 192                 /* count frequency */
 193
 194                 M_MOV_IMM(code, REG_ITMP3);
 195                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 196
 197                 PROFILE_CYCLE_START;
 198         }
 199 #endif
 200
 201         /* create stack frame (if necessary) */
 202
 203         if (cd->stackframesize)
 204                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 205
 206         /* save used callee saved registers */
 207
 208         p = cd->stackframesize;
 209         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 210                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 211         }
 212         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 213                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 214         }
 215
 216         /* take arguments out of register or stack frame */
 217
 218         md = m->parseddesc;
 219
 220         for (p = 0, l = 0; p < md->paramcount; p++) {
 221                 t = md->paramtypes[p].type;
 222
 223                 varindex = jd->local_map[l * 5 + t];
 224
 225                 l++;
 226                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 227                         l++;
 228
 229                 if (varindex == UNUSED)
 230                         continue;
 231
 232                 var = VAR(varindex);
 233
 234                 s1 = md->params[p].regoff;
 235
 236                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 237                         if (!md->params[p].inmemory) {           /* register arguments    */
 238                                 if (!IS_INMEMORY(var->flags))
 239                                         M_INTMOVE(s1, var->vv.regoff);
 240                                 else
 241                                     M_LST(s1, REG_SP, var->vv.regoff);
 242                         }
 243                         else {                                 /* stack arguments       */
 244                                 if (!IS_INMEMORY(var->flags))
 245                                         /* + 8 for return address */
 246                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 247                                 else
 248                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 249                         }
 250                 }
 251                 else {                                     /* floating args         */
 252                         if (!md->params[p].inmemory) {           /* register arguments    */
 253                                 if (!IS_INMEMORY(var->flags))
 254                                         M_FLTMOVE(s1, var->vv.regoff);
 255                                 else
 256                                         M_DST(s1, REG_SP, var->vv.regoff);
 257                         }
 258                         else {                                 /* stack arguments       */
 259                                 if (!IS_INMEMORY(var->flags))
 260                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 261                                 else
 262                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 263                         }
 264                 }
 265         }
 266
 267         /* save monitorenter argument */
 268
 269 #if defined(ENABLE_THREADS)
 270         if (checksync && code_is_synchronized(code)) {
 271                 /* stack offset for monitor argument */
 272
 273                 s1 = rd->memuse;
 274
 275                 if (opt_verbosecall) {
 276                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 277
 278                         for (p = 0; p < INT_ARG_CNT; p++)
 279                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 280
 281                         for (p = 0; p < FLT_ARG_CNT; p++)
 282                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 283
 284                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 285                 }
 286
 287                 /* decide which monitor enter function to call */
 288
 289                 if (m->flags & ACC_STATIC) {
 290                         M_MOV_IMM(&m->class->object.header, REG_A0);
 291                 }
 292                 else {
 293                         M_TEST(REG_A0);
 294                         M_BNE(8);
 295                         M_ALD_MEM(REG_A0, EXCEPTION_HARDWARE_NULLPOINTER);
 296                 }
 297
 298                 M_AST(REG_A0, REG_SP, s1 * 8);
 299                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 300                 M_CALL(REG_ITMP1);
 301
 302                 if (opt_verbosecall) {
 303                         for (p = 0; p < INT_ARG_CNT; p++)
 304                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 305
 306                         for (p = 0; p < FLT_ARG_CNT; p++)
 307                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 308
 309                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 310                 }
 311         }
 312 #endif
 313
 314 #if !defined(NDEBUG)
 315         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 316                 emit_verbosecall_enter(jd);
 317 #endif /* !defined(NDEBUG) */
 318
 319         }
 320
 321         /* end of header generation */
 322
 323         /* create replacement points */
 324
 325         REPLACEMENT_POINTS_INIT(cd, jd);
 326
 327         /* walk through all basic blocks */
 328
 329         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 330
 331                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 332
 333                 if (bptr->flags >= BBREACHED) {
 334
 335                 /* branch resolving */
 336
 337                 codegen_resolve_branchrefs(cd, bptr);
 338
 339                 /* handle replacement points */
 340
 341                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 342
 343                 /* copy interface registers to their destination */
 344
 345                 len = bptr->indepth;
 346                 MCODECHECK(512);
 347
 348 #if defined(ENABLE_PROFILING)
 349                 /* generate basicblock profiling code */
 350
 351                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 352                         /* count frequency */
 353
 354                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 355                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 356
 357                         /* if this is an exception handler, start profiling again */
 358
 359                         if (bptr->type == BBTYPE_EXH)
 360                                 PROFILE_CYCLE_START;
 361                 }
 362 #endif
 363
 364 #if defined(ENABLE_LSRA)
 365                 if (opt_lsra) {
 366                         while (len) {
 367                                 len--;
 368                                 src = bptr->invars[len];
 369                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 370                                         if (bptr->type == BBTYPE_EXH) {
 371 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 372                                                 if (!IS_INMEMORY(src->flags))
 373                                                         d= src->vv.regoff;
 374                                                 else
 375                                                         d=REG_ITMP1;
 376                                                 M_INTMOVE(REG_ITMP1, d);
 377                                                 emit_store(jd, NULL, src, d);
 378                                         }
 379                                 }
 380                         }
 381
 382                 } else {
 383 #endif
 384
 385                 while (len) {
 386                         len--;
 387                         var = VAR(bptr->invars[len]);
 388                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 389                                 if (bptr->type == BBTYPE_EXH) {
 390                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 391                                         M_INTMOVE(REG_ITMP1, d);
 392                                         emit_store(jd, NULL, var, d);
 393                                 }
 394                         }
 395                         else {
 396                                 assert((var->flags & INOUT));
 397                         }
 398                 }
 399 #if defined(ENABLE_LSRA)
 400                 }
 401 #endif
 402                 /* walk through all instructions */
 403
 404                 len = bptr->icount;
 405                 currentline = 0;
 406
 407                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 408                         if (iptr->line != currentline) {
 409                                 dseg_addlinenumber(cd, iptr->line);
 410                                 currentline = iptr->line;
 411                         }
 412
 413                         MCODECHECK(1024);                         /* 1KB should be enough */
 414
 415                 switch (iptr->opc) {
 416                 case ICMD_NOP:        /* ...  ==> ...                                 */
 417                 case ICMD_POP:        /* ..., value  ==> ...                          */
 418                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 419                         break;
 420
 421                 case ICMD_INLINE_START:
 422
 423                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 424                         break;
 425
 426                 case ICMD_INLINE_BODY:
 427
 428                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 429                         dseg_addlinenumber_inline_start(cd, iptr);
 430                         dseg_addlinenumber(cd, iptr->line);
 431                         break;
 432
 433                 case ICMD_INLINE_END:
 434
 435                         dseg_addlinenumber_inline_end(cd, iptr);
 436                         dseg_addlinenumber(cd, iptr->line);
 437                         break;
 438
 439                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 440
 441                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 442                         emit_nullpointer_check(cd, iptr, s1);
 443                         break;
 444
 445                 /* constant operations ************************************************/
 446
 447                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 448
 449                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 450                         ICONST(d, iptr->sx.val.i);
 451                         emit_store_dst(jd, iptr, d);
 452                         break;
 453
 454                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 455
 456                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 457                         LCONST(d, iptr->sx.val.l);
 458                         emit_store_dst(jd, iptr, d);
 459                         break;
 460
 461                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 462
 463                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 464                         disp = dseg_add_float(cd, iptr->sx.val.f);
 465                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 466                         emit_store_dst(jd, iptr, d);
 467                         break;
 468
 469                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 470
 471                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 472                         disp = dseg_add_double(cd, iptr->sx.val.d);
 473                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 474                         emit_store_dst(jd, iptr, d);
 475                         break;
 476
 477                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 478
 479                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 480
 481                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 482                                 cr   = iptr->sx.val.c.ref;
 483                                 disp = dseg_add_unique_address(cd, cr);
 484
 485 /*                              PROFILE_CYCLE_STOP; */
 486
 487                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
 488                                                                           cr, disp);
 489
 490 /*                              PROFILE_CYCLE_START; */
 491
 492                                 M_ALD(d, RIP, disp);
 493                         }
 494                         else {
 495                                 if (iptr->sx.val.anyptr == 0) {
 496                                         M_CLR(d);
 497                                 }
 498                                 else {
 499                                         disp = dseg_add_address(cd, iptr->sx.val.anyptr);
 500                                         M_ALD(d, RIP, disp);
 501                                 }
 502                         }
 503                         emit_store_dst(jd, iptr, d);
 504                         break;
 505
 506
 507                 /* load/store/copy/move operations ************************************/
 508
 509                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 510                 case ICMD_ALOAD:      /* s1 = local variable                          */
 511                 case ICMD_LLOAD:
 512                 case ICMD_FLOAD:
 513                 case ICMD_DLOAD:
 514                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 515                 case ICMD_LSTORE:
 516                 case ICMD_FSTORE:
 517                 case ICMD_DSTORE:
 518                 case ICMD_COPY:
 519                 case ICMD_MOVE:
 520
 521                         emit_copy(jd, iptr);
 522                         break;
 523
 524                 case ICMD_ASTORE:
 525                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 526                                 emit_copy(jd, iptr);
 527                         break;
 528
 529                 /* integer operations *************************************************/
 530
 531                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 532
 533                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 534                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 535                         M_INTMOVE(s1, d);
 536                         M_INEG(d);
 537                         emit_store_dst(jd, iptr, d);
 538                         break;
 539
 540                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 541
 542                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 543                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 544                         M_INTMOVE(s1, d);
 545                         M_LNEG(d);
 546                         emit_store_dst(jd, iptr, d);
 547                         break;
 548
 549                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 550
 551                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 552                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 553                         M_ISEXT(s1, d);
 554                         emit_store_dst(jd, iptr, d);
 555                         break;
 556
 557                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 558
 559                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 560                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 561                         M_IMOV(s1, d);
 562                         emit_store_dst(jd, iptr, d);
 563                         break;
 564
 565                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 566
 567                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 568                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 569                         M_BSEXT(s1, d);
 570                         emit_store_dst(jd, iptr, d);
 571                         break;
 572
 573                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 574
 575                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 576                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 577                         M_CZEXT(s1, d);
 578                         emit_store_dst(jd, iptr, d);
 579                         break;
 580
 581                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 582
 583                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 584                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 585                         M_SSEXT(s1, d);
 586                         emit_store_dst(jd, iptr, d);
 587                         break;
 588
 589
 590                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 591
 592                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 593                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 594                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 595                         if (s2 == d)
 596                                 M_IADD(s1, d);
 597                         else {
 598                                 M_INTMOVE(s1, d);
 599                                 M_IADD(s2, d);
 600                         }
 601                         emit_store_dst(jd, iptr, d);
 602                         break;
 603
 604                 case ICMD_IINC:
 605                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 606                                       /* sx.val.i = constant                             */
 607
 608                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 609                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 610
 611                         /* Using inc and dec is not faster than add (tested with
 612                            sieve). */
 613
 614                         M_INTMOVE(s1, d);
 615                         M_IADD_IMM(iptr->sx.val.i, d);
 616                         emit_store_dst(jd, iptr, d);
 617                         break;
 618
 619                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 620
 621                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 622                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 623                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 624                         if (s2 == d)
 625                                 M_LADD(s1, d);
 626                         else {
 627                                 M_INTMOVE(s1, d);
 628                                 M_LADD(s2, d);
 629                         }
 630                         emit_store_dst(jd, iptr, d);
 631                         break;
 632
 633                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 634                                       /* sx.val.l = constant                             */
 635
 636                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 637                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 638                         M_INTMOVE(s1, d);
 639                         if (IS_IMM32(iptr->sx.val.l))
 640                                 M_LADD_IMM(iptr->sx.val.l, d);
 641                         else {
 642                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 643                                 M_LADD(REG_ITMP2, d);
 644                         }
 645                         emit_store_dst(jd, iptr, d);
 646                         break;
 647
 648                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 649
 650                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 651                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 652                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 653                         if (s2 == d) {
 654                                 M_INTMOVE(s1, REG_ITMP1);
 655                                 M_ISUB(s2, REG_ITMP1);
 656                                 M_INTMOVE(REG_ITMP1, d);
 657                         } else {
 658                                 M_INTMOVE(s1, d);
 659                                 M_ISUB(s2, d);
 660                         }
 661                         emit_store_dst(jd, iptr, d);
 662                         break;
 663
 664                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 665                                       /* sx.val.i = constant                             */
 666
 667                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 668                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 669                         M_INTMOVE(s1, d);
 670                         M_ISUB_IMM(iptr->sx.val.i, d);
 671                         emit_store_dst(jd, iptr, d);
 672                         break;
 673
 674                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 675
 676                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 677                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 678                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 679                         if (s2 == d) {
 680                                 M_INTMOVE(s1, REG_ITMP1);
 681                                 M_LSUB(s2, REG_ITMP1);
 682                                 M_INTMOVE(REG_ITMP1, d);
 683                         } else {
 684                                 M_INTMOVE(s1, d);
 685                                 M_LSUB(s2, d);
 686                         }
 687                         emit_store_dst(jd, iptr, d);
 688                         break;
 689
 690                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 691                                       /* sx.val.l = constant                             */
 692
 693                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 694                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 695                         M_INTMOVE(s1, d);
 696                         if (IS_IMM32(iptr->sx.val.l))
 697                                 M_LSUB_IMM(iptr->sx.val.l, d);
 698                         else {
 699                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 700                                 M_LSUB(REG_ITMP2, d);
 701                         }
 702                         emit_store_dst(jd, iptr, d);
 703                         break;
 704
 705                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 706
 707                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 708                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 709                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 710                         if (s2 == d)
 711                                 M_IMUL(s1, d);
 712                         else {
 713                                 M_INTMOVE(s1, d);
 714                                 M_IMUL(s2, d);
 715                         }
 716                         emit_store_dst(jd, iptr, d);
 717                         break;
 718
 719                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 720                                       /* sx.val.i = constant                             */
 721
 722                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 723                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 724                         if (iptr->sx.val.i == 2) {
 725                                 M_INTMOVE(s1, d);
 726                                 M_ISLL_IMM(1, d);
 727                         } else
 728                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 729                         emit_store_dst(jd, iptr, d);
 730                         break;
 731
 732                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 733
 734                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 735                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 736                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 737                         if (s2 == d)
 738                                 M_LMUL(s1, d);
 739                         else {
 740                                 M_INTMOVE(s1, d);
 741                                 M_LMUL(s2, d);
 742                         }
 743                         emit_store_dst(jd, iptr, d);
 744                         break;
 745
 746                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 747                                       /* sx.val.l = constant                             */
 748
 749                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 750                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 751                         if (IS_IMM32(iptr->sx.val.l))
 752                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 753                         else {
 754                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 755                                 M_INTMOVE(s1, d);
 756                                 M_LMUL(REG_ITMP2, d);
 757                         }
 758                         emit_store_dst(jd, iptr, d);
 759                         break;
 760
 761                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 762
 763                         s1 = emit_load_s1(jd, iptr, RAX);
 764                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 765                         d = codegen_reg_of_dst(jd, iptr, RAX);
 766
 767                         M_INTMOVE(s1, RAX);
 768                         M_INTMOVE(s2, REG_ITMP3);
 769                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 770
 771                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 772
 773                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 774                         M_BNE(4 + 6);
 775                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 776                         M_BEQ(1 + 3);                                          /* 6 bytes */
 777
 778                         emit_cltd(cd);                                         /* 1 byte  */
 779                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 780
 781                         M_INTMOVE(RAX, d);
 782                         emit_store_dst(jd, iptr, d);
 783                         dst = VAROP(iptr->dst);
 784                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 785                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 786                         break;
 787
 788                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 789
 790                         s1 = emit_load_s1(jd, iptr, RAX);
 791                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 792                         d = codegen_reg_of_dst(jd, iptr, RDX);
 793
 794                         M_INTMOVE(s1, RAX);
 795                         M_INTMOVE(s2, REG_ITMP3);
 796                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 797
 798                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 799
 800                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 801                         M_BNE(3 + 4 + 6);
 802                         M_CLR(RDX);                                            /* 3 bytes */
 803                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 804                         M_BEQ(1 + 3);                                          /* 6 bytes */
 805
 806                         emit_cltd(cd);                                         /* 1 byte  */
 807                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 808
 809                         M_INTMOVE(RDX, d);
 810                         emit_store_dst(jd, iptr, d);
 811                         dst = VAROP(iptr->dst);
 812                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 813                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 814                         break;
 815
 816                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 817                                       /* sx.val.i = constant                             */
 818
 819                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 820                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 821                         M_INTMOVE(s1, REG_ITMP1);
 822                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 823                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 824                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 825                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 826                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 827                         emit_store_dst(jd, iptr, d);
 828                         break;
 829
 830                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 831                                       /* sx.val.i = constant                             */
 832
 833                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 834                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 835                         M_INTMOVE(s1, REG_ITMP1);
 836                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 837                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 838                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 839                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 840                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 841                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 842                         emit_store_dst(jd, iptr, d);
 843                         break;
 844
 845
 846                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 847
 848                         s1 = emit_load_s1(jd, iptr, RAX);
 849                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 850                         d = codegen_reg_of_dst(jd, iptr, RAX);
 851
 852                         M_INTMOVE(s1, RAX);
 853                         M_INTMOVE(s2, REG_ITMP3);
 854                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 855
 856                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 857
 858                         /* check as described in jvm spec */
 859                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 860                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 861                         M_BNE(4 + 6);
 862                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 863                         M_BEQ(2 + 3);                                          /* 6 bytes */
 864
 865                         emit_cqto(cd);                                         /* 2 bytes */
 866                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 867
 868                         M_INTMOVE(RAX, d);
 869                         emit_store_dst(jd, iptr, d);
 870                         dst = VAROP(iptr->dst);
 871                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 872                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 873                         break;
 874
 875                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 876
 877                         s1 = emit_load_s1(jd, iptr, RAX);
 878                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 879                         d = codegen_reg_of_dst(jd, iptr, RDX);
 880
 881                         M_INTMOVE(s1, RAX);
 882                         M_INTMOVE(s2, REG_ITMP3);
 883                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 884
 885                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 886
 887                         /* check as described in jvm spec */
 888                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 889                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 890                         M_BNE(3 + 4 + 6);
 891                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 892                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 893                         M_BEQ(2 + 3);                                          /* 6 bytes */
 894
 895                         emit_cqto(cd);                                         /* 2 bytes */
 896                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 897
 898                         M_INTMOVE(RDX, d);
 899                         emit_store_dst(jd, iptr, d);
 900                         dst = VAROP(iptr->dst);
 901                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 902                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 903                         break;
 904
 905                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 906                                       /* sx.val.i = constant                             */
 907
 908                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 909                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 910                         M_INTMOVE(s1, REG_ITMP1);
 911                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 912                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 913                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 914                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 915                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 916                         emit_store_dst(jd, iptr, d);
 917                         break;
 918
 919                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 920                                       /* sx.val.l = constant                             */
 921
 922                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 923                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 924                         M_INTMOVE(s1, REG_ITMP1);
 925                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 926                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 927                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 928                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 929                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 930                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 931                         emit_store_dst(jd, iptr, d);
 932                         break;
 933
 934                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 935
 936                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 937                         emit_ishift(jd, SHIFT_SHL, iptr);
 938                         break;
 939
 940                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 941                                       /* sx.val.i = constant                             */
 942
 943                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 944                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 945                         M_INTMOVE(s1, d);
 946                         M_ISLL_IMM(iptr->sx.val.i, d);
 947                         emit_store_dst(jd, iptr, d);
 948                         break;
 949
 950                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 951
 952                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 953                         emit_ishift(jd, SHIFT_SAR, iptr);
 954                         break;
 955
 956                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 957                                       /* sx.val.i = constant                             */
 958
 959                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 960                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 961                         M_INTMOVE(s1, d);
 962                         M_ISRA_IMM(iptr->sx.val.i, d);
 963                         emit_store_dst(jd, iptr, d);
 964                         break;
 965
 966                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 967
 968                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 969                         emit_ishift(jd, SHIFT_SHR, iptr);
 970                         break;
 971
 972                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 973                                       /* sx.val.i = constant                             */
 974
 975                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 976                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 977                         M_INTMOVE(s1, d);
 978                         M_ISRL_IMM(iptr->sx.val.i, d);
 979                         emit_store_dst(jd, iptr, d);
 980                         break;
 981
 982                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 983
 984                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 985                         emit_lshift(jd, SHIFT_SHL, iptr);
 986                         break;
 987
 988         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 989                                           /* sx.val.i = constant                             */
 990
 991                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 992                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 993                         M_INTMOVE(s1, d);
 994                         M_LSLL_IMM(iptr->sx.val.i, d);
 995                         emit_store_dst(jd, iptr, d);
 996                         break;
 997
 998                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 999
1000                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1001                         emit_lshift(jd, SHIFT_SAR, iptr);
1002                         break;
1003
1004                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1005                                       /* sx.val.i = constant                             */
1006
1007                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1008                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1009                         M_INTMOVE(s1, d);
1010                         M_LSRA_IMM(iptr->sx.val.i, d);
1011                         emit_store_dst(jd, iptr, d);
1012                         break;
1013
1014                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1015
1016                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1017                         emit_lshift(jd, SHIFT_SHR, iptr);
1018                         break;
1019
1020                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1021                                       /* sx.val.l = constant                             */
1022
1023                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1024                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1025                         M_INTMOVE(s1, d);
1026                         M_LSRL_IMM(iptr->sx.val.i, d);
1027                         emit_store_dst(jd, iptr, d);
1028                         break;
1029
1030                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1031
1032                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1033                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1034                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1035                         if (s2 == d)
1036                                 M_IAND(s1, d);
1037                         else {
1038                                 M_INTMOVE(s1, d);
1039                                 M_IAND(s2, d);
1040                         }
1041                         emit_store_dst(jd, iptr, d);
1042                         break;
1043
1044                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1045                                       /* sx.val.i = constant                             */
1046
1047                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1048                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1049                         M_INTMOVE(s1, d);
1050                         M_IAND_IMM(iptr->sx.val.i, d);
1051                         emit_store_dst(jd, iptr, d);
1052                         break;
1053
1054                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1055
1056                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1057                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1058                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1059                         if (s2 == d)
1060                                 M_LAND(s1, d);
1061                         else {
1062                                 M_INTMOVE(s1, d);
1063                                 M_LAND(s2, d);
1064                         }
1065                         emit_store_dst(jd, iptr, d);
1066                         break;
1067
1068                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1069                                       /* sx.val.l = constant                             */
1070
1071                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1072                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1073                         M_INTMOVE(s1, d);
1074                         if (IS_IMM32(iptr->sx.val.l))
1075                                 M_LAND_IMM(iptr->sx.val.l, d);
1076                         else {
1077                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1078                                 M_LAND(REG_ITMP2, d);
1079                         }
1080                         emit_store_dst(jd, iptr, d);
1081                         break;
1082
1083                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1084
1085                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1086                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1087                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1088                         if (s2 == d)
1089                                 M_IOR(s1, d);
1090                         else {
1091                                 M_INTMOVE(s1, d);
1092                                 M_IOR(s2, d);
1093                         }
1094                         emit_store_dst(jd, iptr, d);
1095                         break;
1096
1097                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1098                                       /* sx.val.i = constant                             */
1099
1100                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1101                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1102                         M_INTMOVE(s1, d);
1103                         M_IOR_IMM(iptr->sx.val.i, d);
1104                         emit_store_dst(jd, iptr, d);
1105                         break;
1106
1107                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1108
1109                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1110                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1111                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1112                         if (s2 == d)
1113                                 M_LOR(s1, d);
1114                         else {
1115                                 M_INTMOVE(s1, d);
1116                                 M_LOR(s2, d);
1117                         }
1118                         emit_store_dst(jd, iptr, d);
1119                         break;
1120
1121                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1122                                       /* sx.val.l = constant                             */
1123
1124                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1125                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1126                         M_INTMOVE(s1, d);
1127                         if (IS_IMM32(iptr->sx.val.l))
1128                                 M_LOR_IMM(iptr->sx.val.l, d);
1129                         else {
1130                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1131                                 M_LOR(REG_ITMP2, d);
1132                         }
1133                         emit_store_dst(jd, iptr, d);
1134                         break;
1135
1136                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1137
1138                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1139                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1140                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1141                         if (s2 == d)
1142                                 M_IXOR(s1, d);
1143                         else {
1144                                 M_INTMOVE(s1, d);
1145                                 M_IXOR(s2, d);
1146                         }
1147                         emit_store_dst(jd, iptr, d);
1148                         break;
1149
1150                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1151                                       /* sx.val.i = constant                             */
1152
1153                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1154                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1155                         M_INTMOVE(s1, d);
1156                         M_IXOR_IMM(iptr->sx.val.i, d);
1157                         emit_store_dst(jd, iptr, d);
1158                         break;
1159
1160                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1161
1162                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1163                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1164                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1165                         if (s2 == d)
1166                                 M_LXOR(s1, d);
1167                         else {
1168                                 M_INTMOVE(s1, d);
1169                                 M_LXOR(s2, d);
1170                         }
1171                         emit_store_dst(jd, iptr, d);
1172                         break;
1173
1174                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1175                                       /* sx.val.l = constant                             */
1176
1177                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1178                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1179                         M_INTMOVE(s1, d);
1180                         if (IS_IMM32(iptr->sx.val.l))
1181                                 M_LXOR_IMM(iptr->sx.val.l, d);
1182                         else {
1183                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1184                                 M_LXOR(REG_ITMP2, d);
1185                         }
1186                         emit_store_dst(jd, iptr, d);
1187                         break;
1188
1189
1190                 /* floating operations ************************************************/
1191
1192                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1193
1194                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1195                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1196                         disp = dseg_add_s4(cd, 0x80000000);
1197                         M_FLTMOVE(s1, d);
1198                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1199                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1200                         emit_store_dst(jd, iptr, d);
1201                         break;
1202
1203                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1204
1205                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1206                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1207                         disp = dseg_add_s8(cd, 0x8000000000000000);
1208                         M_FLTMOVE(s1, d);
1209                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1210                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1211                         emit_store_dst(jd, iptr, d);
1212                         break;
1213
1214                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1215
1216                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1217                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1218                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1219                         if (s2 == d)
1220                                 M_FADD(s1, d);
1221                         else {
1222                                 M_FLTMOVE(s1, d);
1223                                 M_FADD(s2, d);
1224                         }
1225                         emit_store_dst(jd, iptr, d);
1226                         break;
1227
1228                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1229
1230                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1231                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1232                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1233                         if (s2 == d)
1234                                 M_DADD(s1, d);
1235                         else {
1236                                 M_FLTMOVE(s1, d);
1237                                 M_DADD(s2, d);
1238                         }
1239                         emit_store_dst(jd, iptr, d);
1240                         break;
1241
1242                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1243
1244                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1245                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1246                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1247                         if (s2 == d) {
1248                                 M_FLTMOVE(s2, REG_FTMP2);
1249                                 s2 = REG_FTMP2;
1250                         }
1251                         M_FLTMOVE(s1, d);
1252                         M_FSUB(s2, d);
1253                         emit_store_dst(jd, iptr, d);
1254                         break;
1255
1256                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1257
1258                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1259                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1260                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1261                         if (s2 == d) {
1262                                 M_FLTMOVE(s2, REG_FTMP2);
1263                                 s2 = REG_FTMP2;
1264                         }
1265                         M_FLTMOVE(s1, d);
1266                         M_DSUB(s2, d);
1267                         emit_store_dst(jd, iptr, d);
1268                         break;
1269
1270                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1271
1272                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1273                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1274                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1275                         if (s2 == d)
1276                                 M_FMUL(s1, d);
1277                         else {
1278                                 M_FLTMOVE(s1, d);
1279                                 M_FMUL(s2, d);
1280                         }
1281                         emit_store_dst(jd, iptr, d);
1282                         break;
1283
1284                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1285
1286                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1287                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1288                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1289                         if (s2 == d)
1290                                 M_DMUL(s1, d);
1291                         else {
1292                                 M_FLTMOVE(s1, d);
1293                                 M_DMUL(s2, d);
1294                         }
1295                         emit_store_dst(jd, iptr, d);
1296                         break;
1297
1298                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1299
1300                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1301                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1302                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1303                         if (s2 == d) {
1304                                 M_FLTMOVE(s2, REG_FTMP2);
1305                                 s2 = REG_FTMP2;
1306                         }
1307                         M_FLTMOVE(s1, d);
1308                         M_FDIV(s2, d);
1309                         emit_store_dst(jd, iptr, d);
1310                         break;
1311
1312                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1313
1314                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1315                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1316                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1317                         if (s2 == d) {
1318                                 M_FLTMOVE(s2, REG_FTMP2);
1319                                 s2 = REG_FTMP2;
1320                         }
1321                         M_FLTMOVE(s1, d);
1322                         M_DDIV(s2, d);
1323                         emit_store_dst(jd, iptr, d);
1324                         break;
1325
1326                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1327
1328                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1329                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1330                         M_CVTIF(s1, d);
1331                         emit_store_dst(jd, iptr, d);
1332                         break;
1333
1334                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1335
1336                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1337                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1338                         M_CVTID(s1, d);
1339                         emit_store_dst(jd, iptr, d);
1340                         break;
1341
1342                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1343
1344                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1345                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1346                         M_CVTLF(s1, d);
1347                         emit_store_dst(jd, iptr, d);
1348                         break;
1349
1350                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1351
1352                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1353                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1354                         M_CVTLD(s1, d);
1355                         emit_store_dst(jd, iptr, d);
1356                         break;
1357
1358                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1359
1360                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1361                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1362                         M_CVTFI(s1, d);
1363                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1364                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1365                                 ((REG_RESULT == d) ? 0 : 3);
1366                         M_BNE(disp);
1367                         M_FLTMOVE(s1, REG_FTMP1);
1368                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1369                         M_CALL(REG_ITMP2);
1370                         M_INTMOVE(REG_RESULT, d);
1371                         emit_store_dst(jd, iptr, d);
1372                         break;
1373
1374                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1375
1376                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1377                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1378                         M_CVTDI(s1, d);
1379                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1380                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1381                                 ((REG_RESULT == d) ? 0 : 3);
1382                         M_BNE(disp);
1383                         M_FLTMOVE(s1, REG_FTMP1);
1384                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1385                         M_CALL(REG_ITMP2);
1386                         M_INTMOVE(REG_RESULT, d);
1387                         emit_store_dst(jd, iptr, d);
1388                         break;
1389
1390                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1391
1392                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1393                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1394                         M_CVTFL(s1, d);
1395                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1396                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1397                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1398                                 ((REG_RESULT == d) ? 0 : 3);
1399                         M_BNE(disp);
1400                         M_FLTMOVE(s1, REG_FTMP1);
1401                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1402                         M_CALL(REG_ITMP2);
1403                         M_INTMOVE(REG_RESULT, d);
1404                         emit_store_dst(jd, iptr, d);
1405                         break;
1406
1407                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1408
1409                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1410                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1411                         M_CVTDL(s1, d);
1412                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1413                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1414                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1415                                 ((REG_RESULT == d) ? 0 : 3);
1416                         M_BNE(disp);
1417                         M_FLTMOVE(s1, REG_FTMP1);
1418                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1419                         M_CALL(REG_ITMP2);
1420                         M_INTMOVE(REG_RESULT, d);
1421                         emit_store_dst(jd, iptr, d);
1422                         break;
1423
1424                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1425
1426                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1427                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1428                         M_CVTFD(s1, d);
1429                         emit_store_dst(jd, iptr, d);
1430                         break;
1431
1432                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1433
1434                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1435                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1436                         M_CVTDF(s1, d);
1437                         emit_store_dst(jd, iptr, d);
1438                         break;
1439
1440                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1441                                           /* == => 0, < => 1, > => -1 */
1442
1443                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1444                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1445                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1446                         M_CLR(d);
1447                         M_MOV_IMM(1, REG_ITMP1);
1448                         M_MOV_IMM(-1, REG_ITMP2);
1449                         emit_ucomiss_reg_reg(cd, s1, s2);
1450                         M_CMOVULT(REG_ITMP1, d);
1451                         M_CMOVUGT(REG_ITMP2, d);
1452                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1453                         emit_store_dst(jd, iptr, d);
1454                         break;
1455
1456                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1457                                           /* == => 0, < => 1, > => -1 */
1458
1459                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1460                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1461                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1462                         M_CLR(d);
1463                         M_MOV_IMM(1, REG_ITMP1);
1464                         M_MOV_IMM(-1, REG_ITMP2);
1465                         emit_ucomiss_reg_reg(cd, s1, s2);
1466                         M_CMOVULT(REG_ITMP1, d);
1467                         M_CMOVUGT(REG_ITMP2, d);
1468                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1469                         emit_store_dst(jd, iptr, d);
1470                         break;
1471
1472                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1473                                           /* == => 0, < => 1, > => -1 */
1474
1475                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1476                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1477                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1478                         M_CLR(d);
1479                         M_MOV_IMM(1, REG_ITMP1);
1480                         M_MOV_IMM(-1, REG_ITMP2);
1481                         emit_ucomisd_reg_reg(cd, s1, s2);
1482                         M_CMOVULT(REG_ITMP1, d);
1483                         M_CMOVUGT(REG_ITMP2, d);
1484                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1485                         emit_store_dst(jd, iptr, d);
1486                         break;
1487
1488                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1489                                           /* == => 0, < => 1, > => -1 */
1490
1491                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1492                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1493                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1494                         M_CLR(d);
1495                         M_MOV_IMM(1, REG_ITMP1);
1496                         M_MOV_IMM(-1, REG_ITMP2);
1497                         emit_ucomisd_reg_reg(cd, s1, s2);
1498                         M_CMOVULT(REG_ITMP1, d);
1499                         M_CMOVUGT(REG_ITMP2, d);
1500                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1501                         emit_store_dst(jd, iptr, d);
1502                         break;
1503
1504
1505                 /* memory operations **************************************************/
1506
1507                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1508
1509                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1510                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1511                         /* implicit null-pointer check */
1512                         M_ILD(d, s1, OFFSET(java_array_t, size));
1513                         emit_store_dst(jd, iptr, d);
1514                         break;
1515
1516                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1517
1518                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1519                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1520                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1521                         /* implicit null-pointer check */
1522                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1523                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1524                         emit_store_dst(jd, iptr, d);
1525                         break;
1526
1527                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1528
1529                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1530                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1531                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1532                         /* implicit null-pointer check */
1533                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1534                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1535                         emit_store_dst(jd, iptr, d);
1536                         break;
1537
1538                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1539
1540                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1541                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1542                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1543                         /* implicit null-pointer check */
1544                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1545                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1546                         emit_store_dst(jd, iptr, d);
1547                         break;
1548
1549                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1550
1551                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1552                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1553                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1554                         /* implicit null-pointer check */
1555                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1556                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1557                         emit_store_dst(jd, iptr, d);
1558                         break;
1559
1560                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1561
1562                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1563                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1564                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1565                         /* implicit null-pointer check */
1566                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1567                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1568                         emit_store_dst(jd, iptr, d);
1569                         break;
1570
1571                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1572
1573                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1574                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1575                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1576                         /* implicit null-pointer check */
1577                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1578                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1579                         emit_store_dst(jd, iptr, d);
1580                         break;
1581
1582                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1583
1584                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1585                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1586                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1587                         /* implicit null-pointer check */
1588                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1589                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1590                         emit_store_dst(jd, iptr, d);
1591                         break;
1592
1593                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1594
1595                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1596                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1597                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1598                         /* implicit null-pointer check */
1599                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1600                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1601                         emit_store_dst(jd, iptr, d);
1602                         break;
1603
1604
1605                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1606
1607                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1608                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1609                         /* implicit null-pointer check */
1610                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1611                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1612                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1613                         break;
1614
1615                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1616
1617                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1618                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1619                         /* implicit null-pointer check */
1620                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1621                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1622                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1623                         break;
1624
1625                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1626
1627                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1628                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1629                         /* implicit null-pointer check */
1630                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1631                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1632                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1633                         break;
1634
1635                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1636
1637                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1638                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1639                         /* implicit null-pointer check */
1640                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1641                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1642                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1643                         break;
1644
1645                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1646
1647                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1648                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1649                         /* implicit null-pointer check */
1650                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1651                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1652                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1653                         break;
1654
1655                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1656
1657                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1658                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1659                         /* implicit null-pointer check */
1660                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1661                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1662                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1663                         break;
1664
1665                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1666
1667                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1668                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1669                         /* implicit null-pointer check */
1670                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1671                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1672                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1673                         break;
1674
1675                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1676
1677                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1678                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1679                         /* implicit null-pointer check */
1680                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1681                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1682
1683                         M_MOV(s1, REG_A0);
1684                         M_MOV(s3, REG_A1);
1685                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
1686                         M_CALL(REG_ITMP1);
1687                         emit_arraystore_check(cd, iptr);
1688
1689                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1690                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1691                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1692                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1693                         break;
1694
1695
1696                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1697
1698                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1699                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1700                         /* implicit null-pointer check */
1701                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1702                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1703                         break;
1704
1705                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1706
1707                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1708                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1709                         /* implicit null-pointer check */
1710                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1711                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1712                         break;
1713
1714                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1715
1716                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1717                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1718                         /* implicit null-pointer check */
1719                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1720                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1721                         break;
1722
1723                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1724
1725                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1726                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1727                         /* implicit null-pointer check */
1728                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1729                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1730                         break;
1731
1732                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1733
1734                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1735                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1736                         /* implicit null-pointer check */
1737                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1738
1739                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1740                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1741                         }
1742                         else {
1743                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1744                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1745                         }
1746                         break;
1747
1748                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1749
1750                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1751                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1752                         /* implicit null-pointer check */
1753                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1754                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1755                         break;
1756
1757
1758                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1759
1760                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1761                                 uf        = iptr->sx.s23.s3.uf;
1762                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1763                                 disp      = dseg_add_unique_address(cd, uf);
1764
1765 /*                              PROFILE_CYCLE_STOP; */
1766
1767                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1768
1769 /*                              PROFILE_CYCLE_START; */
1770                         }
1771                         else {
1772                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1773                                 fieldtype = fi->type;
1774                                 disp      = dseg_add_address(cd, fi->value);
1775
1776                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1777                                         PROFILE_CYCLE_STOP;
1778
1779                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1780                                                                                   fi->class, 0);
1781
1782                                         PROFILE_CYCLE_START;
1783                                 }
1784                         }
1785
1786                         /* This approach is much faster than moving the field
1787                            address inline into a register. */
1788
1789                         M_ALD(REG_ITMP1, RIP, disp);
1790
1791                         switch (fieldtype) {
1792                         case TYPE_INT:
1793                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1794                                 M_ILD(d, REG_ITMP1, 0);
1795                                 break;
1796                         case TYPE_LNG:
1797                         case TYPE_ADR:
1798                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1799                                 M_LLD(d, REG_ITMP1, 0);
1800                                 break;
1801                         case TYPE_FLT:
1802                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1803                                 M_FLD(d, REG_ITMP1, 0);
1804                                 break;
1805                         case TYPE_DBL:
1806                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1807                                 M_DLD(d, REG_ITMP1, 0);
1808                                 break;
1809                         }
1810                         emit_store_dst(jd, iptr, d);
1811                         break;
1812
1813                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1814
1815                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1816                                 uf        = iptr->sx.s23.s3.uf;
1817                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1818                                 disp      = dseg_add_unique_address(cd, uf);
1819
1820 /*                              PROFILE_CYCLE_STOP; */
1821
1822                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1823
1824 /*                              PROFILE_CYCLE_START; */
1825                         }
1826                         else {
1827                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1828                                 fieldtype = fi->type;
1829                                 disp      = dseg_add_address(cd, fi->value);
1830
1831                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1832                                         PROFILE_CYCLE_STOP;
1833
1834                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1835                                                                                   fi->class, 0);
1836
1837                                         PROFILE_CYCLE_START;
1838                                 }
1839                         }
1840
1841                         /* This approach is much faster than moving the field
1842                            address inline into a register. */
1843
1844                         M_ALD(REG_ITMP1, RIP, disp);
1845
1846                         switch (fieldtype) {
1847                         case TYPE_INT:
1848                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1849                                 M_IST(s1, REG_ITMP1, 0);
1850                                 break;
1851                         case TYPE_LNG:
1852                         case TYPE_ADR:
1853                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1854                                 M_LST(s1, REG_ITMP1, 0);
1855                                 break;
1856                         case TYPE_FLT:
1857                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1858                                 M_FST(s1, REG_ITMP1, 0);
1859                                 break;
1860                         case TYPE_DBL:
1861                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1862                                 M_DST(s1, REG_ITMP1, 0);
1863                                 break;
1864                         }
1865                         break;
1866
1867                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1868                                           /* val = value (in current instruction)     */
1869                                           /* following NOP)                           */
1870
1871                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1872                                 uf        = iptr->sx.s23.s3.uf;
1873                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1874                                 disp      = dseg_add_unique_address(cd, uf);
1875
1876 /*                              PROFILE_CYCLE_STOP; */
1877
1878                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1879
1880 /*                              PROFILE_CYCLE_START; */
1881                         }
1882                         else {
1883                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1884                                 fieldtype = fi->type;
1885                                 disp      = dseg_add_address(cd, fi->value);
1886
1887                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1888                                         PROFILE_CYCLE_STOP;
1889
1890                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1891                                                                                   fi->class, 0);
1892
1893                                         PROFILE_CYCLE_START;
1894                                 }
1895                         }
1896
1897                         /* This approach is much faster than moving the field
1898                            address inline into a register. */
1899
1900                         M_ALD(REG_ITMP1, RIP, disp);
1901
1902                         switch (fieldtype) {
1903                         case TYPE_INT:
1904                         case TYPE_FLT:
1905                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1906                                 break;
1907                         case TYPE_LNG:
1908                         case TYPE_ADR:
1909                         case TYPE_DBL:
1910                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1911                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1912                                 else {
1913                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1914                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1915                                 }
1916                                 break;
1917                         }
1918                         break;
1919
1920                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1921
1922                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1923
1924                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1925                                 uf        = iptr->sx.s23.s3.uf;
1926                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1927                                 disp      = 0;
1928
1929 /*                              PROFILE_CYCLE_STOP; */
1930
1931                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1932
1933 /*                              PROFILE_CYCLE_START; */
1934                         }
1935                         else {
1936                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1937                                 fieldtype = fi->type;
1938                                 disp      = fi->offset;
1939                         }
1940
1941                         /* implicit null-pointer check */
1942                         switch (fieldtype) {
1943                         case TYPE_INT:
1944                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1945                                 M_ILD32(d, s1, disp);
1946                                 break;
1947                         case TYPE_LNG:
1948                         case TYPE_ADR:
1949                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1950                                 M_LLD32(d, s1, disp);
1951                                 break;
1952                         case TYPE_FLT:
1953                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1954                                 M_FLD32(d, s1, disp);
1955                                 break;
1956                         case TYPE_DBL:
1957                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1958                                 M_DLD32(d, s1, disp);
1959                                 break;
1960                         }
1961                         emit_store_dst(jd, iptr, d);
1962                         break;
1963
1964                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1965
1966                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1967                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1968
1969                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1970                                 uf        = iptr->sx.s23.s3.uf;
1971                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1972                                 disp      = 0;
1973
1974 /*                              PROFILE_CYCLE_STOP; */
1975
1976                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1977
1978 /*                              PROFILE_CYCLE_START; */
1979                         }
1980                         else {
1981                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1982                                 fieldtype = fi->type;
1983                                 disp      = fi->offset;
1984                         }
1985
1986                         /* implicit null-pointer check */
1987                         switch (fieldtype) {
1988                         case TYPE_INT:
1989                                 M_IST32(s2, s1, disp);
1990                                 break;
1991                         case TYPE_LNG:
1992                         case TYPE_ADR:
1993                                 M_LST32(s2, s1, disp);
1994                                 break;
1995                         case TYPE_FLT:
1996                                 M_FST32(s2, s1, disp);
1997                                 break;
1998                         case TYPE_DBL:
1999                                 M_DST32(s2, s1, disp);
2000                                 break;
2001                         }
2002                         break;
2003
2004                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2005                                           /* val = value (in current instruction)     */
2006                                           /* following NOP)                           */
2007
2008                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2009
2010                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2011                                 uf        = iptr->sx.s23.s3.uf;
2012                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2013                                 disp      = 0;
2014
2015 /*                              PROFILE_CYCLE_STOP; */
2016
2017                                 patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
2018
2019 /*                              PROFILE_CYCLE_START; */
2020                         }
2021                         else {
2022                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2023                                 fieldtype = fi->type;
2024                                 disp      = fi->offset;
2025                         }
2026
2027                         /* implicit null-pointer check */
2028                         switch (fieldtype) {
2029                         case TYPE_INT:
2030                         case TYPE_FLT:
2031                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2032                                 break;
2033                         case TYPE_LNG:
2034                         case TYPE_ADR:
2035                         case TYPE_DBL:
2036                                 /* XXX why no check for IS_IMM32? */
2037                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2038                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2039                                 break;
2040                         }
2041                         break;
2042
2043
2044                 /* branch operations **************************************************/
2045
2046                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2047
2048                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2049                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2050
2051                         PROFILE_CYCLE_STOP;
2052
2053 #ifdef ENABLE_VERIFIER
2054                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2055                                 uc = iptr->sx.s23.s2.uc;
2056
2057                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2058                         }
2059 #endif /* ENABLE_VERIFIER */
2060
2061                         M_CALL_IMM(0);                            /* passing exception pc */
2062                         M_POP(REG_ITMP2_XPC);
2063
2064                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2065                         M_JMP(REG_ITMP3);
2066                         break;
2067
2068                 case ICMD_GOTO:         /* ... ==> ...                                */
2069                 case ICMD_RET:
2070
2071                         emit_br(cd, iptr->dst.block);
2072                         ALIGNCODENOP;
2073                         break;
2074
2075                 case ICMD_JSR:          /* ... ==> ...                                */
2076
2077                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2078                         ALIGNCODENOP;
2079                         break;
2080
2081                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2082                 case ICMD_IFNONNULL:
2083
2084                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2085                         M_TEST(s1);
2086                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2087                         break;
2088
2089                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2090                 case ICMD_IFLT:
2091                 case ICMD_IFLE:
2092                 case ICMD_IFNE:
2093                 case ICMD_IFGT:
2094                 case ICMD_IFGE:
2095
2096                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2097                         M_ICMP_IMM(iptr->sx.val.i, s1);
2098                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2099                         break;
2100
2101                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2102                 case ICMD_IF_LNE:
2103                 case ICMD_IF_LLT:
2104                 case ICMD_IF_LGE:
2105                 case ICMD_IF_LGT:
2106                 case ICMD_IF_LLE:
2107
2108                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2109                         if (IS_IMM32(iptr->sx.val.l))
2110                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2111                         else {
2112                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2113                                 M_LCMP(REG_ITMP2, s1);
2114                         }
2115                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2116                         break;
2117
2118                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2119                 case ICMD_IF_ICMPNE:
2120                 case ICMD_IF_ICMPLT:
2121                 case ICMD_IF_ICMPGE:
2122                 case ICMD_IF_ICMPGT:
2123                 case ICMD_IF_ICMPLE:
2124
2125                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2126                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2127                         M_ICMP(s2, s1);
2128                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2129                         break;
2130
2131                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2132                 case ICMD_IF_ACMPNE:
2133
2134                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2135                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2136                         M_LCMP(s2, s1);
2137                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2138                         break;
2139
2140                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2141                 case ICMD_IF_LCMPNE:
2142                 case ICMD_IF_LCMPLT:
2143                 case ICMD_IF_LCMPGE:
2144                 case ICMD_IF_LCMPGT:
2145                 case ICMD_IF_LCMPLE:
2146
2147                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2148                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2149                         M_LCMP(s2, s1);
2150                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2151                         break;
2152
2153                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2154                 case ICMD_LRETURN:
2155
2156                         REPLACEMENT_POINT_RETURN(cd, iptr);
2157                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2158                         M_INTMOVE(s1, REG_RESULT);
2159                         goto nowperformreturn;
2160
2161                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2162
2163                         REPLACEMENT_POINT_RETURN(cd, iptr);
2164                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2165                         M_INTMOVE(s1, REG_RESULT);
2166
2167 #ifdef ENABLE_VERIFIER
2168                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2169                                 uc = iptr->sx.s23.s2.uc;
2170
2171                                 PROFILE_CYCLE_STOP;
2172
2173                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2174
2175                                 PROFILE_CYCLE_START;
2176                         }
2177 #endif /* ENABLE_VERIFIER */
2178                         goto nowperformreturn;
2179
2180                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2181                 case ICMD_DRETURN:
2182
2183                         REPLACEMENT_POINT_RETURN(cd, iptr);
2184                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2185                         M_FLTMOVE(s1, REG_FRESULT);
2186                         goto nowperformreturn;
2187
2188                 case ICMD_RETURN:      /* ...  ==> ...                                */
2189
2190                         REPLACEMENT_POINT_RETURN(cd, iptr);
2191
2192 nowperformreturn:
2193                         {
2194                         s4 i, p;
2195
2196                         p = cd->stackframesize;
2197
2198 #if !defined(NDEBUG)
2199                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2200                                 emit_verbosecall_exit(jd);
2201 #endif /* !defined(NDEBUG) */
2202
2203 #if defined(ENABLE_THREADS)
2204                         if (checksync && code_is_synchronized(code)) {
2205                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2206
2207                                 /* we need to save the proper return value */
2208                                 switch (iptr->opc) {
2209                                 case ICMD_IRETURN:
2210                                 case ICMD_ARETURN:
2211                                 case ICMD_LRETURN:
2212                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2213                                         break;
2214                                 case ICMD_FRETURN:
2215                                 case ICMD_DRETURN:
2216                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2217                                         break;
2218                                 }
2219
2220                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2221                                 M_CALL(REG_ITMP1);
2222
2223                                 /* and now restore the proper return value */
2224                                 switch (iptr->opc) {
2225                                 case ICMD_IRETURN:
2226                                 case ICMD_ARETURN:
2227                                 case ICMD_LRETURN:
2228                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2229                                         break;
2230                                 case ICMD_FRETURN:
2231                                 case ICMD_DRETURN:
2232                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2233                                         break;
2234                                 }
2235                         }
2236 #endif
2237
2238                         /* restore saved registers */
2239
2240                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2241                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2242                         }
2243                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2244                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2245                         }
2246
2247                         /* deallocate stack */
2248
2249                         if (cd->stackframesize)
2250                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2251
2252                         /* generate method profiling code */
2253
2254                         PROFILE_CYCLE_STOP;
2255
2256                         M_RET;
2257                         }
2258                         break;
2259
2260
2261                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2262                         {
2263                                 s4 i, l;
2264                                 branch_target_t *table;
2265
2266                                 table = iptr->dst.table;
2267
2268                                 l = iptr->sx.s23.s2.tablelow;
2269                                 i = iptr->sx.s23.s3.tablehigh;
2270
2271                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2272                                 M_INTMOVE(s1, REG_ITMP1);
2273
2274                                 if (l != 0)
2275                                         M_ISUB_IMM(l, REG_ITMP1);
2276
2277                                 /* number of targets */
2278                                 i = i - l + 1;
2279
2280                 /* range check */
2281
2282                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2283                                 emit_bugt(cd, table[0].block);
2284
2285                                 /* build jump table top down and use address of lowest entry */
2286
2287                                 table += i;
2288
2289                                 while (--i >= 0) {
2290                                         dseg_add_target(cd, table->block);
2291                                         --table;
2292                                 }
2293
2294                                 /* length of dataseg after last dseg_add_target is used
2295                                    by load */
2296
2297                                 M_MOV_IMM(0, REG_ITMP2);
2298                                 dseg_adddata(cd);
2299                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2300                                 M_JMP(REG_ITMP1);
2301                         }
2302                         break;
2303
2304
2305                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2306                         {
2307                                 s4 i;
2308                                 lookup_target_t *lookup;
2309
2310                                 lookup = iptr->dst.lookup;
2311
2312                                 i = iptr->sx.s23.s2.lookupcount;
2313
2314                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2315                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2316
2317                                 while (--i >= 0) {
2318                                         M_ICMP_IMM(lookup->value, s1);
2319                                         emit_beq(cd, lookup->target.block);
2320                                         lookup++;
2321                                 }
2322
2323                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2324                                 ALIGNCODENOP;
2325                         }
2326                         break;
2327
2328
2329                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2330
2331                         REPLACEMENT_POINT_FORGC_BUILTIN(cd, iptr);
2332
2333                         bte = iptr->sx.s23.s3.bte;
2334                         md  = bte->md;
2335                         goto gen_method;
2336
2337                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2338
2339                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2340                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2341                 case ICMD_INVOKEINTERFACE:
2342
2343                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2344
2345                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2346                                 lm = NULL;
2347                                 um = iptr->sx.s23.s3.um;
2348                                 md = um->methodref->parseddesc.md;
2349                         }
2350                         else {
2351                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2352                                 um = NULL;
2353                                 md = lm->parseddesc;
2354                         }
2355
2356 gen_method:
2357                         s3 = md->paramcount;
2358
2359                         MCODECHECK((20 * s3) + 128);
2360
2361                         /* copy arguments to registers or stack location */
2362
2363                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2364                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2365                                 d   = md->params[s3].regoff;
2366
2367                                 /* already preallocated (ARGVAR)? */
2368
2369                                 if (var->flags & PREALLOC)
2370                                         continue;
2371
2372                                 if (IS_INT_LNG_TYPE(var->type)) {
2373                                         if (!md->params[s3].inmemory) {
2374                                                 s1 = emit_load(jd, iptr, var, d);
2375                                                 M_INTMOVE(s1, d);
2376                                         }
2377                                         else {
2378                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2379                                                 M_LST(s1, REG_SP, d);
2380                                         }
2381                                 }
2382                                 else {
2383                                         if (!md->params[s3].inmemory) {
2384                                                 s1 = emit_load(jd, iptr, var, d);
2385                                                 M_FLTMOVE(s1, d);
2386                                         }
2387                                         else {
2388                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2389
2390                                                 if (IS_2_WORD_TYPE(var->type))
2391                                                         M_DST(s1, REG_SP, d);
2392                                                 else
2393                                                         M_FST(s1, REG_SP, d);
2394                                         }
2395                                 }
2396                         }
2397
2398                         /* generate method profiling code */
2399
2400                         PROFILE_CYCLE_STOP;
2401
2402                         switch (iptr->opc) {
2403                         case ICMD_BUILTIN:
2404                                 if (bte->stub == NULL) {
2405                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2406                                 }
2407                                 else {
2408                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2409                                 }
2410                                 M_CALL(REG_ITMP1);
2411                                 break;
2412
2413                         case ICMD_INVOKESPECIAL:
2414                                 emit_nullpointer_check(cd, iptr, REG_A0);
2415                                 /* fall through */
2416
2417                         case ICMD_INVOKESTATIC:
2418                                 if (lm == NULL) {
2419                                         disp = dseg_add_unique_address(cd, um);
2420
2421                                         patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
2422                                                                                   um, disp);
2423                                 }
2424                                 else {
2425                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2426                                 }
2427
2428                                 M_ALD(REG_ITMP2, RIP, disp);
2429                                 M_CALL(REG_ITMP2);
2430                                 break;
2431
2432                         case ICMD_INVOKEVIRTUAL:
2433                                 if (lm == NULL) {
2434                                         patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
2435
2436                                         s1 = 0;
2437                                 }
2438                                 else {
2439                                         s1 = OFFSET(vftbl_t, table[0]) +
2440                                                 sizeof(methodptr) * lm->vftblindex;
2441                                 }
2442
2443                                 /* implicit null-pointer check */
2444                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2445                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2446                                 M_CALL(REG_ITMP3);
2447                                 break;
2448
2449                         case ICMD_INVOKEINTERFACE:
2450                                 if (lm == NULL) {
2451                                         patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
2452
2453                                         s1 = 0;
2454                                         s2 = 0;
2455                                 }
2456                                 else {
2457                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2458                                                 sizeof(methodptr) * lm->class->index;
2459
2460                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2461                                 }
2462
2463                                 /* implicit null-pointer check */
2464                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2465                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2466                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2467                                 M_CALL(REG_ITMP3);
2468                                 break;
2469                         }
2470
2471                         /* generate method profiling code */
2472
2473                         PROFILE_CYCLE_START;
2474
2475                         /* store size of call code in replacement point */
2476
2477                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2478                         REPLACEMENT_POINT_FORGC_BUILTIN_RETURN(cd, iptr);
2479
2480                         /* store return value */
2481
2482                         switch (md->returntype.type) {
2483                         case TYPE_INT:
2484                         case TYPE_LNG:
2485                         case TYPE_ADR:
2486                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2487                                 M_INTMOVE(REG_RESULT, s1);
2488                                 emit_store_dst(jd, iptr, s1);
2489                                 break;
2490                         case TYPE_FLT:
2491                         case TYPE_DBL:
2492                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2493                                 M_FLTMOVE(REG_FRESULT, s1);
2494                                 emit_store_dst(jd, iptr, s1);
2495                                 break;
2496                         default:
2497                                 /* TYPE_VOID */
2498                                 break;
2499                         }
2500                         break;
2501
2502
2503                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2504
2505                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2506                                 /* object type cast-check */
2507
2508                                 classinfo *super;
2509                                 s4         superindex;
2510
2511                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2512                                         super      = NULL;
2513                                         superindex = 0;
2514                                 }
2515                                 else {
2516                                         super      = iptr->sx.s23.s3.c.cls;
2517                                         superindex = super->index;
2518                                 }
2519
2520                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2521                                         CODEGEN_CRITICAL_SECTION_NEW;
2522
2523                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2524
2525                                 /* if class is not resolved, check which code to call */
2526
2527                                 if (super == NULL) {
2528                                         M_TEST(s1);
2529                                         emit_label_beq(cd, BRANCH_LABEL_1);
2530
2531                                         patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2532                                                                                   iptr->sx.s23.s3.c.ref, 0);
2533
2534                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2535                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2536                                         emit_label_beq(cd, BRANCH_LABEL_2);
2537                                 }
2538
2539                                 /* interface checkcast code */
2540
2541                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2542                                         if (super != NULL) {
2543                                                 M_TEST(s1);
2544                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2545                                         }
2546
2547                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2548
2549                                         if (super == NULL) {
2550                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
2551                                                                                           iptr->sx.s23.s3.c.ref,
2552                                                                                           0);
2553                                         }
2554
2555                                         M_ILD32(REG_ITMP3,
2556                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2557                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2558                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2559
2560                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2561                                                         OFFSET(vftbl_t, interfacetable[0]) -
2562                                                         superindex * sizeof(methodptr*));
2563                                         M_TEST(REG_ITMP3);
2564                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2565
2566                                         if (super == NULL)
2567                                                 emit_label_br(cd, BRANCH_LABEL_4);
2568                                         else
2569                                                 emit_label(cd, BRANCH_LABEL_3);
2570                                 }
2571
2572                                 /* class checkcast code */
2573
2574                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2575                                         if (super == NULL) {
2576                                                 emit_label(cd, BRANCH_LABEL_2);
2577
2578                                                 cr   = iptr->sx.s23.s3.c.ref;
2579                                                 disp = dseg_add_unique_address(cd, cr);
2580
2581                                                 patcher_add_patch_ref(jd,
2582                                                                                           PATCHER_resolve_classref_to_vftbl,
2583                                                                                           cr, disp);
2584                                         }
2585                                         else {
2586                                                 M_TEST(s1);
2587                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2588
2589                                                 disp = dseg_add_address(cd, super->vftbl);
2590                                         }
2591
2592                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2593                                         M_ALD(REG_ITMP3, RIP, disp);
2594
2595                                         CODEGEN_CRITICAL_SECTION_START;
2596
2597                                         M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2598
2599                                         /*                                      if (s1 != REG_ITMP1) { */
2600                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2601                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2602                                         /*                                                                                              REG_ITMP1); */
2603                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2604                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2605                                         /*                                                                                              REG_ITMP3); */
2606                                         /*  #if defined(ENABLE_THREADS) */
2607                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2608                                         /*  #endif */
2609                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2610
2611                                         /*                                      } else { */
2612
2613                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
2614                                         M_ISUB(REG_ITMP3, REG_ITMP2);
2615                                         M_ALD(REG_ITMP3, RIP, disp);
2616                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2617                                         /*                                      } */
2618
2619                                         CODEGEN_CRITICAL_SECTION_END;
2620
2621                                         M_ICMP(REG_ITMP3, REG_ITMP2);
2622                                         emit_classcast_check(cd, iptr, BRANCH_UGT, REG_ITMP3, s1);
2623
2624                                         if (super != NULL)
2625                                                 emit_label(cd, BRANCH_LABEL_5);
2626                                 }
2627
2628                                 if (super == NULL) {
2629                                         emit_label(cd, BRANCH_LABEL_1);
2630                                         emit_label(cd, BRANCH_LABEL_4);
2631                                 }
2632
2633                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2634                         }
2635                         else {
2636                                 /* array type cast-check */
2637
2638                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2639                                 M_INTMOVE(s1, REG_A0);
2640
2641                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2642                                         cr   = iptr->sx.s23.s3.c.ref;
2643                                         disp = dseg_add_unique_address(cd, cr);
2644
2645                                         patcher_add_patch_ref(jd,
2646                                                                                   PATCHER_resolve_classref_to_classinfo,
2647                                                                                   cr, disp);
2648                                 }
2649                                 else {
2650                                         disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2651                                 }
2652
2653                                 M_ALD(REG_A1, RIP, disp);
2654                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2655                                 M_CALL(REG_ITMP1);
2656
2657                                 /* s1 may have been destroyed over the function call */
2658                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2659                                 M_TEST(REG_RESULT);
2660                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2661
2662                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2663                         }
2664
2665                         M_INTMOVE(s1, d);
2666                         emit_store_dst(jd, iptr, d);
2667                         break;
2668
2669                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2670
2671                         {
2672                         classinfo *super;
2673                         s4         superindex;
2674
2675                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2676                                 super      = NULL;
2677                                 superindex = 0;
2678                         }
2679                         else {
2680                                 super      = iptr->sx.s23.s3.c.cls;
2681                                 superindex = super->index;
2682                         }
2683
2684                         if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2685                                 CODEGEN_CRITICAL_SECTION_NEW;
2686
2687                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2688                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2689
2690                         if (s1 == d) {
2691                                 M_INTMOVE(s1, REG_ITMP1);
2692                                 s1 = REG_ITMP1;
2693                         }
2694
2695                         M_CLR(d);
2696
2697                         /* if class is not resolved, check which code to call */
2698
2699                         if (super == NULL) {
2700                                 M_TEST(s1);
2701                                 emit_label_beq(cd, BRANCH_LABEL_1);
2702
2703                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2704                                                                           iptr->sx.s23.s3.c.ref, 0);
2705
2706                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2707                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2708                                 emit_label_beq(cd, BRANCH_LABEL_2);
2709                         }
2710
2711                         /* interface instanceof code */
2712
2713                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2714                                 if (super != NULL) {
2715                                         M_TEST(s1);
2716                                         emit_label_beq(cd, BRANCH_LABEL_3);
2717                                 }
2718
2719                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2720
2721                                 if (super == NULL) {
2722                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
2723                                                                                   iptr->sx.s23.s3.c.ref, 0);
2724                                 }
2725
2726                                 M_ILD32(REG_ITMP3,
2727                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2728                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2729
2730                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2731
2732                                 M_BLE(a);
2733                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2734                                                 OFFSET(vftbl_t, interfacetable[0]) -
2735                                                 superindex * sizeof(methodptr*));
2736                                 M_TEST(REG_ITMP1);
2737                                 M_SETNE(d);
2738
2739                                 if (super == NULL)
2740                                         emit_label_br(cd, BRANCH_LABEL_4);
2741                                 else
2742                                         emit_label(cd, BRANCH_LABEL_3);
2743                         }
2744
2745                         /* class instanceof code */
2746
2747                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2748                                 if (super == NULL) {
2749                                         emit_label(cd, BRANCH_LABEL_2);
2750
2751                                         cr   = iptr->sx.s23.s3.c.ref;
2752                                         disp = dseg_add_unique_address(cd, cr);
2753
2754                                         patcher_add_patch_ref(jd,
2755                                                                                   PATCHER_resolve_classref_to_vftbl,
2756                                                                                   cr, disp);
2757                                 }
2758                                 else {
2759                                         M_TEST(s1);
2760                                         emit_label_beq(cd, BRANCH_LABEL_5);
2761
2762                                         disp = dseg_add_address(cd, super->vftbl);
2763                                 }
2764
2765                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2766                                 M_ALD(REG_ITMP2, RIP, disp);
2767
2768                                 CODEGEN_CRITICAL_SECTION_START;
2769
2770                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
2771                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2772                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2773
2774                                 CODEGEN_CRITICAL_SECTION_END;
2775
2776                                 M_ISUB(REG_ITMP2, REG_ITMP1);
2777                                 M_CLR(d); /* may be REG_ITMP2 */
2778                                 M_ICMP(REG_ITMP3, REG_ITMP1);
2779                                 M_SETULE(d);
2780
2781                                 if (super != NULL)
2782                                         emit_label(cd, BRANCH_LABEL_5);
2783                         }
2784
2785                         if (super == NULL) {
2786                                 emit_label(cd, BRANCH_LABEL_1);
2787                                 emit_label(cd, BRANCH_LABEL_4);
2788                         }
2789
2790                         emit_store_dst(jd, iptr, d);
2791                         }
2792                         break;
2793
2794                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2795
2796                         /* check for negative sizes and copy sizes to stack if necessary  */
2797
2798                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2799
2800                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2801
2802                                 /* copy SAVEDVAR sizes to stack */
2803                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2804
2805                                 /* Already Preallocated? */
2806                                 if (!(var->flags & PREALLOC)) {
2807                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2808                                         M_LST(s2, REG_SP, s1 * 8);
2809                                 }
2810                         }
2811
2812                         /* a0 = dimension count */
2813
2814                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2815
2816                         /* is a patcher function set? */
2817
2818                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2819                                 cr   = iptr->sx.s23.s3.c.ref;
2820                                 disp = dseg_add_unique_address(cd, cr);
2821
2822                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
2823                                                                           cr, disp);
2824                         }
2825                         else {
2826                                 disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2827                         }
2828
2829                         /* a1 = classinfo */
2830
2831                         M_ALD(REG_A1, RIP, disp);
2832
2833                         /* a2 = pointer to dimensions = stack pointer */
2834
2835                         M_MOV(REG_SP, REG_A2);
2836
2837                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2838                         M_CALL(REG_ITMP1);
2839
2840                         /* check for exception before result assignment */
2841
2842                         emit_exception_check(cd, iptr);
2843
2844                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2845                         M_INTMOVE(REG_RESULT, s1);
2846                         emit_store_dst(jd, iptr, s1);
2847                         break;
2848
2849                 default:
2850                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2851                                                                                    iptr->opc);
2852                         return false;
2853         } /* switch */
2854
2855         } /* for instruction */
2856
2857         MCODECHECK(512); /* XXX require a lower number? */
2858
2859         /* At the end of a basic block we may have to append some nops,
2860            because the patcher stub calling code might be longer than the
2861            actual instruction. So codepatching does not change the
2862            following block unintentionally. */
2863
2864         if (cd->mcodeptr < cd->lastmcodeptr) {
2865                 while (cd->mcodeptr < cd->lastmcodeptr) {
2866                         M_NOP;
2867                 }
2868         }
2869
2870         } /* if (bptr -> flags >= BBREACHED) */
2871         } /* for basic block */
2872
2873         dseg_createlinenumbertable(cd);
2874
2875         /* Generate patcher traps. */
2876
2877         emit_patcher_traps(jd);
2878
2879         /* everything's ok */
2880
2881         return true;
2882 }
2883
2884
2885 /* codegen_emit_stub_native ****************************************************
2886
2887    Emits a stub routine which calls a native method.
2888
2889 *******************************************************************************/
2890
2891 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f, int skipparams)
2892 {
2893         methodinfo  *m;
2894         codeinfo    *code;
2895         codegendata *cd;
2896         methoddesc  *md;
2897         int          i, j;
2898         int          s1, s2;
2899         int          disp;
2900
2901         /* Sanity check. */
2902
2903         assert(f != NULL);
2904
2905         /* Get required compiler data. */
2906
2907         m    = jd->m;
2908         code = jd->code;
2909         cd   = jd->cd;
2910
2911         /* initialize variables */
2912
2913         md = m->parseddesc;
2914
2915         /* calculate stack frame size */
2916
2917         cd->stackframesize =
2918                 sizeof(stackframeinfo_t) / SIZEOF_VOID_P +
2919                 sizeof(localref_table) / SIZEOF_VOID_P +
2920                 md->paramcount +
2921                 (md->returntype.type == TYPE_VOID ? 0 : 1) +
2922                 nmd->memuse;
2923
2924         ALIGN_ODD(cd->stackframesize);              /* keep stack 16-byte aligned */
2925
2926         /* create method header */
2927
2928         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2929         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2930         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
2931         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2932         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2933         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2934         (void) dseg_addlinenumbertablesize(cd);
2935         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
2936
2937 #if defined(ENABLE_PROFILING)
2938         /* generate native method profiling code */
2939
2940         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
2941                 /* count frequency */
2942
2943                 M_MOV_IMM(code, REG_ITMP3);
2944                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
2945         }
2946 #endif
2947
2948         /* generate stub code */
2949
2950         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2951
2952 #if defined(ENABLE_GC_CACAO)
2953         /* Save callee saved integer registers in stackframeinfo (GC may
2954            need to recover them during a collection). */
2955
2956         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
2957                 OFFSET(stackframeinfo_t, intregs);
2958
2959         for (i = 0; i < INT_SAV_CNT; i++)
2960                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2961 #endif
2962
2963         /* save integer and float argument registers */
2964
2965         for (i = 0; i < md->paramcount; i++) {
2966                 if (!md->params[i].inmemory) {
2967                         s1 = md->params[i].regoff;
2968
2969                         switch (md->paramtypes[i].type) {
2970                         case TYPE_INT:
2971                         case TYPE_LNG:
2972                         case TYPE_ADR:
2973                                 M_LST(s1, REG_SP, i * 8);
2974                                 break;
2975                         case TYPE_FLT:
2976                         case TYPE_DBL:
2977                                 M_DST(s1, REG_SP, i * 8);
2978                                 break;
2979                         }
2980                 }
2981         }
2982
2983         /* create dynamic stack info */
2984
2985         M_MOV(REG_SP, REG_A0);
2986         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
2987         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
2988         M_CALL(REG_ITMP1);
2989
2990         /* remember class argument */
2991
2992         if (m->flags & ACC_STATIC)
2993                 M_MOV(REG_RESULT, REG_ITMP2);
2994
2995         /* restore integer and float argument registers */
2996
2997         for (i = 0; i < md->paramcount; i++) {
2998                 if (!md->params[i].inmemory) {
2999                         s1 = md->params[i].regoff;
3000
3001                         switch (md->paramtypes[i].type) {
3002                         case TYPE_INT:
3003                         case TYPE_LNG:
3004                         case TYPE_ADR:
3005                                 M_LLD(s1, REG_SP, i * 8);
3006                                 break;
3007                         case TYPE_FLT:
3008                         case TYPE_DBL:
3009                                 M_DLD(s1, REG_SP, i * 8);
3010                                 break;
3011                         }
3012                 }
3013         }
3014
3015         /* Copy or spill arguments to new locations. */
3016
3017         for (i = md->paramcount - 1, j = i + skipparams; i >= 0; i--, j--) {
3018                 s2 = nmd->params[j].regoff;
3019
3020                 switch (md->paramtypes[i].type) {
3021                 case TYPE_INT:
3022                 case TYPE_LNG:
3023                 case TYPE_ADR:
3024                         if (!md->params[i].inmemory) {
3025                                 s1 = md->params[i].regoff;
3026
3027                                 if (!nmd->params[j].inmemory)
3028                                         M_INTMOVE(s1, s2);
3029                                 else
3030                                         M_LST(s1, REG_SP, s2);
3031                         }
3032                         else {
3033                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3034                                 M_LLD(REG_ITMP1, REG_SP, s1);
3035                                 M_LST(REG_ITMP1, REG_SP, s2);
3036                         }
3037                         break;
3038                 case TYPE_FLT:
3039                         /* We only copy spilled float arguments, as the float
3040                            argument registers keep unchanged. */
3041
3042                         if (md->params[i].inmemory) {
3043                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3044
3045                                 M_FLD(REG_FTMP1, REG_SP, s1);
3046                                 M_FST(REG_FTMP1, REG_SP, s2);
3047                         }
3048                         break;
3049                 case TYPE_DBL:
3050                         if (md->params[i].inmemory) {
3051                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3052                                 M_DLD(REG_FTMP1, REG_SP, s1);
3053                                 M_DST(REG_FTMP1, REG_SP, s2);
3054                         }
3055                         break;
3056                 }
3057         }
3058
3059         /* Handle native Java methods. */
3060
3061         if (m->flags & ACC_NATIVE) {
3062                 /* put class into second argument register */
3063
3064                 if (m->flags & ACC_STATIC)
3065                         M_MOV(REG_ITMP2, REG_A1);
3066
3067                 /* put env into first argument register */
3068
3069                 M_MOV_IMM(_Jv_env, REG_A0);
3070         }
3071
3072         /* Call the native function. */
3073
3074         disp = dseg_add_functionptr(cd, f);
3075         M_ALD(REG_ITMP1, RIP, disp);
3076         M_CALL(REG_ITMP1);
3077
3078         /* save return value */
3079
3080         switch (md->returntype.type) {
3081         case TYPE_INT:
3082         case TYPE_LNG:
3083         case TYPE_ADR:
3084                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3085                 break;
3086         case TYPE_FLT:
3087         case TYPE_DBL:
3088                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3089                 break;
3090         case TYPE_VOID:
3091                 break;
3092         }
3093
3094         /* remove native stackframe info */
3095
3096         M_MOV(REG_SP, REG_A0);
3097         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3098         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3099         M_CALL(REG_ITMP1);
3100         M_MOV(REG_RESULT, REG_ITMP3);
3101
3102         /* restore return value */
3103
3104         switch (md->returntype.type) {
3105         case TYPE_INT:
3106         case TYPE_LNG:
3107         case TYPE_ADR:
3108                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3109                 break;
3110         case TYPE_FLT:
3111         case TYPE_DBL:
3112                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3113                 break;
3114         case TYPE_VOID:
3115                 break;
3116         }
3117
3118 #if defined(ENABLE_GC_CACAO)
3119         /* Restore callee saved integer registers from stackframeinfo (GC
3120            might have modified them during a collection). */
3121
3122         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
3123                 OFFSET(stackframeinfo_t, intregs);
3124
3125         for (i = 0; i < INT_SAV_CNT; i++)
3126                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3127 #endif
3128
3129         /* remove stackframe */
3130
3131         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3132
3133         /* test for exception */
3134
3135         M_TEST(REG_ITMP3);
3136         M_BNE(1);
3137         M_RET;
3138
3139         /* handle exception */
3140
3141         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3142         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3143         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3144
3145         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3146         M_JMP(REG_ITMP3);
3147 }
3148
3149
3150 /*
3151  * These are local overrides for various environment variables in Emacs.
3152  * Please do not remove this and leave it at the end of the file, where
3153  * Emacs will automagically detect them.
3154  * ---------------------------------------------------------------------
3155  * Local variables:
3156  * mode: c
3157  * indent-tabs-mode: t
3158  * c-basic-offset: 4
3159  * tab-width: 4
3160  * End:
3161  * vim:noexpandtab:sw=4:ts=4:
3162  */