src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    $Id: codegen.c 7866 2007-05-03 22:32:11Z twisti $
  26
  27 */
  28
  29
  30 #include "config.h"
  31
  32 #include <assert.h>
  33 #include <stdio.h>
  34
  35 #include "vm/types.h"
  36
  37 #include "md-abi.h"
  38
  39 #include "vm/jit/x86_64/arch.h"
  40 #include "vm/jit/x86_64/codegen.h"
  41 #include "vm/jit/x86_64/emit.h"
  42
  43 #include "mm/memory.h"
  44
  45 #include "native/jni.h"
  46 #include "native/native.h"
  47
  48 #include "threads/lock-common.h"
  49
  50 #include "vm/builtin.h"
  51 #include "vm/exceptions.h"
  52 #include "vm/global.h"
  53 #include "vm/stringlocal.h"
  54 #include "vm/vm.h"
  55
  56 #include "vm/jit/abi.h"
  57 #include "vm/jit/asmpart.h"
  58 #include "vm/jit/codegen-common.h"
  59 #include "vm/jit/dseg.h"
  60 #include "vm/jit/emit-common.h"
  61 #include "vm/jit/jit.h"
  62 #include "vm/jit/methodheader.h"
  63 #include "vm/jit/parse.h"
  64 #include "vm/jit/patcher.h"
  65 #include "vm/jit/reg.h"
  66 #include "vm/jit/replace.h"
  67 #include "vm/jit/stacktrace.h"
  68
  69 #if defined(ENABLE_LSRA)
  70 # include "vm/jit/allocator/lsra.h"
  71 #endif
  72
  73 #include "vmcore/loader.h"
  74 #include "vmcore/options.h"
  75 #include "vmcore/statistics.h"
  76
  77
  78 /* codegen_emit ****************************************************************
  79
  80    Generates machine code.
  81
  82 *******************************************************************************/
  83
  84 bool codegen_emit(jitdata *jd)
  85 {
  86         methodinfo         *m;
  87         codeinfo           *code;
  88         codegendata        *cd;
  89         registerdata       *rd;
  90         s4                  len, s1, s2, s3, d, disp;
  91         u2                  currentline;
  92         ptrint              a;
  93         varinfo            *var, *dst;
  94         basicblock         *bptr;
  95         instruction        *iptr;
  96         exception_entry    *ex;
  97         constant_classref  *cr;
  98         unresolved_class   *uc;
  99         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 100         unresolved_method  *um;
 101         builtintable_entry *bte;
 102         methoddesc         *md;
 103         fieldinfo          *fi;
 104         unresolved_field   *uf;
 105         s4                  fieldtype;
 106         s4                 varindex;
 107
 108         /* get required compiler data */
 109
 110         m    = jd->m;
 111         code = jd->code;
 112         cd   = jd->cd;
 113         rd   = jd->rd;
 114
 115         /* prevent compiler warnings */
 116
 117         d   = 0;
 118         lm  = NULL;
 119         um  = NULL;
 120         bte = NULL;
 121
 122         {
 123         s4 i, p, t, l;
 124         s4 savedregs_num;
 125
 126         savedregs_num = 0;
 127
 128         /* space to save used callee saved registers */
 129
 130         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 131         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 132
 133         cd->stackframesize = rd->memuse + savedregs_num;
 134
 135 #if defined(ENABLE_THREADS)
 136         /* space to save argument of monitor_enter */
 137
 138         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 139                 cd->stackframesize++;
 140 #endif
 141
 142         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 143            native code e.g. libc or jni (alignment problems with
 144            movaps). */
 145
 146         if (!jd->isleafmethod || opt_verbosecall)
 147                 cd->stackframesize |= 0x1;
 148
 149         /* create method header */
 150
 151         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 152         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 153
 154 #if defined(ENABLE_THREADS)
 155         /* IsSync contains the offset relative to the stack pointer for the
 156            argument of monitor_exit used in the exception handler. Since the
 157            offset could be zero and give a wrong meaning of the flag it is
 158            offset by one.
 159         */
 160
 161         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 162                 (void) dseg_add_unique_s4(cd, (rd->memuse + 1) * 8); /* IsSync        */
 163         else
 164 #endif
 165                 (void) dseg_add_unique_s4(cd, 0);                  /* IsSync          */
 166
 167         (void) dseg_add_unique_s4(cd, jd->isleafmethod);               /* IsLeaf  */
 168         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 169         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 170
 171         (void) dseg_addlinenumbertablesize(cd);
 172
 173         (void) dseg_add_unique_s4(cd, jd->exceptiontablelength); /* ExTableSize   */
 174
 175         /* create exception table */
 176
 177         for (ex = jd->exceptiontable; ex != NULL; ex = ex->down) {
 178                 dseg_add_target(cd, ex->start);
 179                 dseg_add_target(cd, ex->end);
 180                 dseg_add_target(cd, ex->handler);
 181                 (void) dseg_add_unique_address(cd, ex->catchtype.any);
 182         }
 183
 184 #if defined(ENABLE_PROFILING)
 185         /* generate method profiling code */
 186
 187         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 188                 /* count frequency */
 189
 190                 M_MOV_IMM(code, REG_ITMP3);
 191                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 192
 193                 PROFILE_CYCLE_START;
 194         }
 195 #endif
 196
 197         /* create stack frame (if necessary) */
 198
 199         if (cd->stackframesize)
 200                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 201
 202         /* save used callee saved registers */
 203
 204         p = cd->stackframesize;
 205         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 206                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 207         }
 208         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 209                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 210         }
 211
 212         /* take arguments out of register or stack frame */
 213
 214         md = m->parseddesc;
 215
 216         for (p = 0, l = 0; p < md->paramcount; p++) {
 217                 t = md->paramtypes[p].type;
 218
 219                 varindex = jd->local_map[l * 5 + t];
 220
 221                 l++;
 222                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 223                         l++;
 224
 225                 if (varindex == UNUSED)
 226                         continue;
 227
 228                 var = VAR(varindex);
 229
 230                 s1 = md->params[p].regoff;
 231
 232                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 233                         if (!md->params[p].inmemory) {           /* register arguments    */
 234                                 if (!IS_INMEMORY(var->flags))
 235                                         M_INTMOVE(s1, var->vv.regoff);
 236                                 else
 237                                     M_LST(s1, REG_SP, var->vv.regoff * 8);
 238                         }
 239                         else {                                 /* stack arguments       */
 240                                 if (!IS_INMEMORY(var->flags))
 241                                         /* + 8 for return address */
 242                                         M_LLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 243                                 else
 244                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 245                         }
 246                 }
 247                 else {                                     /* floating args         */
 248                         if (!md->params[p].inmemory) {           /* register arguments    */
 249                                 if (!IS_INMEMORY(var->flags))
 250                                         M_FLTMOVE(s1, var->vv.regoff);
 251                                 else
 252                                         M_DST(s1, REG_SP, var->vv.regoff * 8);
 253                         }
 254                         else {                                 /* stack arguments       */
 255                                 if (!IS_INMEMORY(var->flags))
 256                                         M_DLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 257                                 else
 258                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 259                         }
 260                 }
 261         }
 262
 263         /* save monitorenter argument */
 264
 265 #if defined(ENABLE_THREADS)
 266         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 267                 /* stack offset for monitor argument */
 268
 269                 s1 = rd->memuse;
 270
 271                 if (opt_verbosecall) {
 272                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 273
 274                         for (p = 0; p < INT_ARG_CNT; p++)
 275                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 276
 277                         for (p = 0; p < FLT_ARG_CNT; p++)
 278                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 279
 280                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 281                 }
 282
 283                 /* decide which monitor enter function to call */
 284
 285                 if (m->flags & ACC_STATIC) {
 286                         M_MOV_IMM(&m->class->object.header, REG_A0);
 287                 }
 288                 else {
 289                         M_TEST(REG_A0);
 290                         M_BNE(8);
 291                         M_ALD_MEM(REG_A0, EXCEPTION_HARDWARE_NULLPOINTER);
 292                 }
 293
 294                 M_AST(REG_A0, REG_SP, s1 * 8);
 295                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 296                 M_CALL(REG_ITMP1);
 297
 298                 if (opt_verbosecall) {
 299                         for (p = 0; p < INT_ARG_CNT; p++)
 300                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 301
 302                         for (p = 0; p < FLT_ARG_CNT; p++)
 303                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 304
 305                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 306                 }
 307         }
 308 #endif
 309
 310 #if !defined(NDEBUG)
 311         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 312                 emit_verbosecall_enter(jd);
 313 #endif /* !defined(NDEBUG) */
 314
 315         }
 316
 317         /* end of header generation */
 318
 319         /* create replacement points */
 320
 321         REPLACEMENT_POINTS_INIT(cd, jd);
 322
 323         /* walk through all basic blocks */
 324
 325         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 326
 327                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 328
 329                 if (bptr->flags >= BBREACHED) {
 330
 331                 /* branch resolving */
 332
 333                 codegen_resolve_branchrefs(cd, bptr);
 334
 335                 /* handle replacement points */
 336
 337                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 338
 339                 /* copy interface registers to their destination */
 340
 341                 len = bptr->indepth;
 342                 MCODECHECK(512);
 343
 344 #if defined(ENABLE_PROFILING)
 345                 /* generate basicblock profiling code */
 346
 347                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 348                         /* count frequency */
 349
 350                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 351                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 352
 353                         /* if this is an exception handler, start profiling again */
 354
 355                         if (bptr->type == BBTYPE_EXH)
 356                                 PROFILE_CYCLE_START;
 357                 }
 358 #endif
 359
 360 #if defined(ENABLE_LSRA)
 361                 if (opt_lsra) {
 362                         while (len) {
 363                                 len--;
 364                                 src = bptr->invars[len];
 365                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 366                                         if (bptr->type == BBTYPE_EXH) {
 367 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 368                                                 if (!IS_INMEMORY(src->flags))
 369                                                         d= src->vv.regoff;
 370                                                 else
 371                                                         d=REG_ITMP1;
 372                                                 M_INTMOVE(REG_ITMP1, d);
 373                                                 emit_store(jd, NULL, src, d);
 374                                         }
 375                                 }
 376                         }
 377
 378                 } else {
 379 #endif
 380
 381                 while (len) {
 382                         len--;
 383                         var = VAR(bptr->invars[len]);
 384                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 385                                 if (bptr->type == BBTYPE_EXH) {
 386                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 387                                         M_INTMOVE(REG_ITMP1, d);
 388                                         emit_store(jd, NULL, var, d);
 389                                 }
 390                         }
 391                         else {
 392                                 assert((var->flags & INOUT));
 393                         }
 394                 }
 395 #if defined(ENABLE_LSRA)
 396                 }
 397 #endif
 398                 /* walk through all instructions */
 399
 400                 len = bptr->icount;
 401                 currentline = 0;
 402
 403                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 404                         if (iptr->line != currentline) {
 405                                 dseg_addlinenumber(cd, iptr->line);
 406                                 currentline = iptr->line;
 407                         }
 408
 409                         MCODECHECK(1024);                         /* 1KB should be enough */
 410
 411                 switch (iptr->opc) {
 412                 case ICMD_NOP:        /* ...  ==> ...                                 */
 413                 case ICMD_POP:        /* ..., value  ==> ...                          */
 414                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 415                         break;
 416
 417                 case ICMD_INLINE_START:
 418
 419                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 420                         break;
 421
 422                 case ICMD_INLINE_BODY:
 423
 424                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 425                         dseg_addlinenumber_inline_start(cd, iptr);
 426                         dseg_addlinenumber(cd, iptr->line);
 427                         break;
 428
 429                 case ICMD_INLINE_END:
 430
 431                         dseg_addlinenumber_inline_end(cd, iptr);
 432                         dseg_addlinenumber(cd, iptr->line);
 433                         break;
 434
 435                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 436
 437                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 438                         emit_nullpointer_check(cd, iptr, s1);
 439                         break;
 440
 441                 /* constant operations ************************************************/
 442
 443                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 444
 445                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 446                         ICONST(d, iptr->sx.val.i);
 447                         emit_store_dst(jd, iptr, d);
 448                         break;
 449
 450                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 451
 452                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 453                         LCONST(d, iptr->sx.val.l);
 454                         emit_store_dst(jd, iptr, d);
 455                         break;
 456
 457                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 458
 459                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 460                         disp = dseg_add_float(cd, iptr->sx.val.f);
 461                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 462                         emit_store_dst(jd, iptr, d);
 463                         break;
 464
 465                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 466
 467                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 468                         disp = dseg_add_double(cd, iptr->sx.val.d);
 469                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 470                         emit_store_dst(jd, iptr, d);
 471                         break;
 472
 473                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 474
 475                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 476
 477                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 478                                 cr = iptr->sx.val.c.ref;
 479
 480 /*                              PROFILE_CYCLE_STOP; */
 481
 482                                 codegen_add_patch_ref(cd, PATCHER_aconst, cr, 0);
 483
 484 /*                              PROFILE_CYCLE_START; */
 485
 486                                 M_MOV_IMM(NULL, d);
 487
 488                         } else {
 489                                 if (iptr->sx.val.anyptr == 0)
 490                                         M_CLR(d);
 491                                 else
 492                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 493                         }
 494                         emit_store_dst(jd, iptr, d);
 495                         break;
 496
 497
 498                 /* load/store/copy/move operations ************************************/
 499
 500                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 501                 case ICMD_ALOAD:      /* s1 = local variable                          */
 502                 case ICMD_LLOAD:
 503                 case ICMD_FLOAD:
 504                 case ICMD_DLOAD:
 505                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 506                 case ICMD_LSTORE:
 507                 case ICMD_FSTORE:
 508                 case ICMD_DSTORE:
 509                 case ICMD_COPY:
 510                 case ICMD_MOVE:
 511
 512                         emit_copy(jd, iptr);
 513                         break;
 514
 515                 case ICMD_ASTORE:
 516                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 517                                 emit_copy(jd, iptr);
 518                         break;
 519
 520                 /* integer operations *************************************************/
 521
 522                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 523
 524                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 525                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 526                         M_INTMOVE(s1, d);
 527                         M_INEG(d);
 528                         emit_store_dst(jd, iptr, d);
 529                         break;
 530
 531                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 532
 533                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 534                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 535                         M_INTMOVE(s1, d);
 536                         M_LNEG(d);
 537                         emit_store_dst(jd, iptr, d);
 538                         break;
 539
 540                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 541
 542                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 543                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 544                         M_ISEXT(s1, d);
 545                         emit_store_dst(jd, iptr, d);
 546                         break;
 547
 548                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 549
 550                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 551                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 552                         M_IMOV(s1, d);
 553                         emit_store_dst(jd, iptr, d);
 554                         break;
 555
 556                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 557
 558                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 559                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 560                         M_BSEXT(s1, d);
 561                         emit_store_dst(jd, iptr, d);
 562                         break;
 563
 564                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 565
 566                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 567                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 568                         M_CZEXT(s1, d);
 569                         emit_store_dst(jd, iptr, d);
 570                         break;
 571
 572                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 573
 574                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 575                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 576                         M_SSEXT(s1, d);
 577                         emit_store_dst(jd, iptr, d);
 578                         break;
 579
 580
 581                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 582
 583                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 584                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 585                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 586                         if (s2 == d)
 587                                 M_IADD(s1, d);
 588                         else {
 589                                 M_INTMOVE(s1, d);
 590                                 M_IADD(s2, d);
 591                         }
 592                         emit_store_dst(jd, iptr, d);
 593                         break;
 594
 595                 case ICMD_IINC:
 596                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 597                                       /* sx.val.i = constant                             */
 598
 599                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 600                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 601
 602                         /* Using inc and dec is not faster than add (tested with
 603                            sieve). */
 604
 605                         M_INTMOVE(s1, d);
 606                         M_IADD_IMM(iptr->sx.val.i, d);
 607                         emit_store_dst(jd, iptr, d);
 608                         break;
 609
 610                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 611
 612                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 613                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 614                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 615                         if (s2 == d)
 616                                 M_LADD(s1, d);
 617                         else {
 618                                 M_INTMOVE(s1, d);
 619                                 M_LADD(s2, d);
 620                         }
 621                         emit_store_dst(jd, iptr, d);
 622                         break;
 623
 624                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 625                                       /* sx.val.l = constant                             */
 626
 627                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 628                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 629                         M_INTMOVE(s1, d);
 630                         if (IS_IMM32(iptr->sx.val.l))
 631                                 M_LADD_IMM(iptr->sx.val.l, d);
 632                         else {
 633                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 634                                 M_LADD(REG_ITMP2, d);
 635                         }
 636                         emit_store_dst(jd, iptr, d);
 637                         break;
 638
 639                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 640
 641                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 642                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 643                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 644                         if (s2 == d) {
 645                                 M_INTMOVE(s1, REG_ITMP1);
 646                                 M_ISUB(s2, REG_ITMP1);
 647                                 M_INTMOVE(REG_ITMP1, d);
 648                         } else {
 649                                 M_INTMOVE(s1, d);
 650                                 M_ISUB(s2, d);
 651                         }
 652                         emit_store_dst(jd, iptr, d);
 653                         break;
 654
 655                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 656                                       /* sx.val.i = constant                             */
 657
 658                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 659                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 660                         M_INTMOVE(s1, d);
 661                         M_ISUB_IMM(iptr->sx.val.i, d);
 662                         emit_store_dst(jd, iptr, d);
 663                         break;
 664
 665                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 666
 667                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 668                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 669                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 670                         if (s2 == d) {
 671                                 M_INTMOVE(s1, REG_ITMP1);
 672                                 M_LSUB(s2, REG_ITMP1);
 673                                 M_INTMOVE(REG_ITMP1, d);
 674                         } else {
 675                                 M_INTMOVE(s1, d);
 676                                 M_LSUB(s2, d);
 677                         }
 678                         emit_store_dst(jd, iptr, d);
 679                         break;
 680
 681                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 682                                       /* sx.val.l = constant                             */
 683
 684                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 685                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 686                         M_INTMOVE(s1, d);
 687                         if (IS_IMM32(iptr->sx.val.l))
 688                                 M_LSUB_IMM(iptr->sx.val.l, d);
 689                         else {
 690                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 691                                 M_LSUB(REG_ITMP2, d);
 692                         }
 693                         emit_store_dst(jd, iptr, d);
 694                         break;
 695
 696                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 697
 698                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 699                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 700                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 701                         if (s2 == d)
 702                                 M_IMUL(s1, d);
 703                         else {
 704                                 M_INTMOVE(s1, d);
 705                                 M_IMUL(s2, d);
 706                         }
 707                         emit_store_dst(jd, iptr, d);
 708                         break;
 709
 710                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 711                                       /* sx.val.i = constant                             */
 712
 713                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 714                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 715                         if (iptr->sx.val.i == 2) {
 716                                 M_INTMOVE(s1, d);
 717                                 M_ISLL_IMM(1, d);
 718                         } else
 719                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 720                         emit_store_dst(jd, iptr, d);
 721                         break;
 722
 723                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 724
 725                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 726                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 727                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 728                         if (s2 == d)
 729                                 M_LMUL(s1, d);
 730                         else {
 731                                 M_INTMOVE(s1, d);
 732                                 M_LMUL(s2, d);
 733                         }
 734                         emit_store_dst(jd, iptr, d);
 735                         break;
 736
 737                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 738                                       /* sx.val.l = constant                             */
 739
 740                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 741                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 742                         if (IS_IMM32(iptr->sx.val.l))
 743                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 744                         else {
 745                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 746                                 M_INTMOVE(s1, d);
 747                                 M_LMUL(REG_ITMP2, d);
 748                         }
 749                         emit_store_dst(jd, iptr, d);
 750                         break;
 751
 752                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 753
 754                         s1 = emit_load_s1(jd, iptr, RAX);
 755                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 756                         d = codegen_reg_of_dst(jd, iptr, RAX);
 757
 758                         M_INTMOVE(s1, RAX);
 759                         M_INTMOVE(s2, REG_ITMP3);
 760                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 761
 762                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 763
 764                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 765                         M_BNE(4 + 6);
 766                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 767                         M_BEQ(1 + 3);                                          /* 6 bytes */
 768
 769                         emit_cltd(cd);                                         /* 1 byte  */
 770                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 771
 772                         M_INTMOVE(RAX, d);
 773                         emit_store_dst(jd, iptr, d);
 774                         dst = VAROP(iptr->dst);
 775                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 776                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 777                         break;
 778
 779                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 780
 781                         s1 = emit_load_s1(jd, iptr, RAX);
 782                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 783                         d = codegen_reg_of_dst(jd, iptr, RDX);
 784
 785                         M_INTMOVE(s1, RAX);
 786                         M_INTMOVE(s2, REG_ITMP3);
 787                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 788
 789                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 790
 791                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 792                         M_BNE(3 + 4 + 6);
 793                         M_CLR(RDX);                                            /* 3 bytes */
 794                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 795                         M_BEQ(1 + 3);                                          /* 6 bytes */
 796
 797                         emit_cltd(cd);                                         /* 1 byte  */
 798                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 799
 800                         M_INTMOVE(RDX, d);
 801                         emit_store_dst(jd, iptr, d);
 802                         dst = VAROP(iptr->dst);
 803                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 804                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 805                         break;
 806
 807                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 808                                       /* sx.val.i = constant                             */
 809
 810                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 811                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 812                         M_INTMOVE(s1, REG_ITMP1);
 813                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 814                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 815                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 816                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 817                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 818                         emit_store_dst(jd, iptr, d);
 819                         break;
 820
 821                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 822                                       /* sx.val.i = constant                             */
 823
 824                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 825                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 826                         M_INTMOVE(s1, REG_ITMP1);
 827                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 828                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 829                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 830                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 831                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 832                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 833                         emit_store_dst(jd, iptr, d);
 834                         break;
 835
 836
 837                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 838
 839                         s1 = emit_load_s1(jd, iptr, RAX);
 840                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 841                         d = codegen_reg_of_dst(jd, iptr, RAX);
 842
 843                         M_INTMOVE(s1, RAX);
 844                         M_INTMOVE(s2, REG_ITMP3);
 845                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 846
 847                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 848
 849                         /* check as described in jvm spec */
 850                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 851                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 852                         M_BNE(4 + 6);
 853                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 854                         M_BEQ(2 + 3);                                          /* 6 bytes */
 855
 856                         emit_cqto(cd);                                         /* 2 bytes */
 857                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 858
 859                         M_INTMOVE(RAX, d);
 860                         emit_store_dst(jd, iptr, d);
 861                         dst = VAROP(iptr->dst);
 862                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 863                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 864                         break;
 865
 866                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 867
 868                         s1 = emit_load_s1(jd, iptr, RAX);
 869                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 870                         d = codegen_reg_of_dst(jd, iptr, RDX);
 871
 872                         M_INTMOVE(s1, RAX);
 873                         M_INTMOVE(s2, REG_ITMP3);
 874                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 875
 876                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 877
 878                         /* check as described in jvm spec */
 879                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 880                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 881                         M_BNE(3 + 4 + 6);
 882                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 883                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 884                         M_BEQ(2 + 3);                                          /* 6 bytes */
 885
 886                         emit_cqto(cd);                                         /* 2 bytes */
 887                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 888
 889                         M_INTMOVE(RDX, d);
 890                         emit_store_dst(jd, iptr, d);
 891                         dst = VAROP(iptr->dst);
 892                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 893                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 894                         break;
 895
 896                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 897                                       /* sx.val.i = constant                             */
 898
 899                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 900                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 901                         M_INTMOVE(s1, REG_ITMP1);
 902                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 903                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 904                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 905                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 906                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 907                         emit_store_dst(jd, iptr, d);
 908                         break;
 909
 910                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 911                                       /* sx.val.l = constant                             */
 912
 913                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 914                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 915                         M_INTMOVE(s1, REG_ITMP1);
 916                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 917                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 918                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 919                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 920                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 921                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 922                         emit_store_dst(jd, iptr, d);
 923                         break;
 924
 925                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 926
 927                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 928                         emit_ishift(jd, SHIFT_SHL, iptr);
 929                         break;
 930
 931                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 932                                       /* sx.val.i = constant                             */
 933
 934                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 935                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 936                         M_INTMOVE(s1, d);
 937                         M_ISLL_IMM(iptr->sx.val.i, d);
 938                         emit_store_dst(jd, iptr, d);
 939                         break;
 940
 941                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 942
 943                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 944                         emit_ishift(jd, SHIFT_SAR, iptr);
 945                         break;
 946
 947                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 948                                       /* sx.val.i = constant                             */
 949
 950                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 951                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 952                         M_INTMOVE(s1, d);
 953                         M_ISRA_IMM(iptr->sx.val.i, d);
 954                         emit_store_dst(jd, iptr, d);
 955                         break;
 956
 957                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 958
 959                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 960                         emit_ishift(jd, SHIFT_SHR, iptr);
 961                         break;
 962
 963                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 964                                       /* sx.val.i = constant                             */
 965
 966                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 967                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 968                         M_INTMOVE(s1, d);
 969                         M_ISRL_IMM(iptr->sx.val.i, d);
 970                         emit_store_dst(jd, iptr, d);
 971                         break;
 972
 973                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 974
 975                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 976                         emit_lshift(jd, SHIFT_SHL, iptr);
 977                         break;
 978
 979         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 980                                           /* sx.val.i = constant                             */
 981
 982                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 983                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 984                         M_INTMOVE(s1, d);
 985                         M_LSLL_IMM(iptr->sx.val.i, d);
 986                         emit_store_dst(jd, iptr, d);
 987                         break;
 988
 989                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 990
 991                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 992                         emit_lshift(jd, SHIFT_SAR, iptr);
 993                         break;
 994
 995                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 996                                       /* sx.val.i = constant                             */
 997
 998                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 999                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1000                         M_INTMOVE(s1, d);
1001                         M_LSRA_IMM(iptr->sx.val.i, d);
1002                         emit_store_dst(jd, iptr, d);
1003                         break;
1004
1005                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1006
1007                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1008                         emit_lshift(jd, SHIFT_SHR, iptr);
1009                         break;
1010
1011                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1012                                       /* sx.val.l = constant                             */
1013
1014                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1015                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1016                         M_INTMOVE(s1, d);
1017                         M_LSRL_IMM(iptr->sx.val.i, d);
1018                         emit_store_dst(jd, iptr, d);
1019                         break;
1020
1021                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1022
1023                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1024                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1025                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1026                         if (s2 == d)
1027                                 M_IAND(s1, d);
1028                         else {
1029                                 M_INTMOVE(s1, d);
1030                                 M_IAND(s2, d);
1031                         }
1032                         emit_store_dst(jd, iptr, d);
1033                         break;
1034
1035                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1036                                       /* sx.val.i = constant                             */
1037
1038                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1039                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1040                         M_INTMOVE(s1, d);
1041                         M_IAND_IMM(iptr->sx.val.i, d);
1042                         emit_store_dst(jd, iptr, d);
1043                         break;
1044
1045                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1046
1047                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1048                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1049                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1050                         if (s2 == d)
1051                                 M_LAND(s1, d);
1052                         else {
1053                                 M_INTMOVE(s1, d);
1054                                 M_LAND(s2, d);
1055                         }
1056                         emit_store_dst(jd, iptr, d);
1057                         break;
1058
1059                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1060                                       /* sx.val.l = constant                             */
1061
1062                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1063                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1064                         M_INTMOVE(s1, d);
1065                         if (IS_IMM32(iptr->sx.val.l))
1066                                 M_LAND_IMM(iptr->sx.val.l, d);
1067                         else {
1068                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1069                                 M_LAND(REG_ITMP2, d);
1070                         }
1071                         emit_store_dst(jd, iptr, d);
1072                         break;
1073
1074                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1075
1076                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1077                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1078                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1079                         if (s2 == d)
1080                                 M_IOR(s1, d);
1081                         else {
1082                                 M_INTMOVE(s1, d);
1083                                 M_IOR(s2, d);
1084                         }
1085                         emit_store_dst(jd, iptr, d);
1086                         break;
1087
1088                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1089                                       /* sx.val.i = constant                             */
1090
1091                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1092                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1093                         M_INTMOVE(s1, d);
1094                         M_IOR_IMM(iptr->sx.val.i, d);
1095                         emit_store_dst(jd, iptr, d);
1096                         break;
1097
1098                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1099
1100                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1101                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1102                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1103                         if (s2 == d)
1104                                 M_LOR(s1, d);
1105                         else {
1106                                 M_INTMOVE(s1, d);
1107                                 M_LOR(s2, d);
1108                         }
1109                         emit_store_dst(jd, iptr, d);
1110                         break;
1111
1112                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1113                                       /* sx.val.l = constant                             */
1114
1115                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1116                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1117                         M_INTMOVE(s1, d);
1118                         if (IS_IMM32(iptr->sx.val.l))
1119                                 M_LOR_IMM(iptr->sx.val.l, d);
1120                         else {
1121                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1122                                 M_LOR(REG_ITMP2, d);
1123                         }
1124                         emit_store_dst(jd, iptr, d);
1125                         break;
1126
1127                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1128
1129                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1130                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1131                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1132                         if (s2 == d)
1133                                 M_IXOR(s1, d);
1134                         else {
1135                                 M_INTMOVE(s1, d);
1136                                 M_IXOR(s2, d);
1137                         }
1138                         emit_store_dst(jd, iptr, d);
1139                         break;
1140
1141                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1142                                       /* sx.val.i = constant                             */
1143
1144                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1145                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1146                         M_INTMOVE(s1, d);
1147                         M_IXOR_IMM(iptr->sx.val.i, d);
1148                         emit_store_dst(jd, iptr, d);
1149                         break;
1150
1151                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1152
1153                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1154                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1155                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1156                         if (s2 == d)
1157                                 M_LXOR(s1, d);
1158                         else {
1159                                 M_INTMOVE(s1, d);
1160                                 M_LXOR(s2, d);
1161                         }
1162                         emit_store_dst(jd, iptr, d);
1163                         break;
1164
1165                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1166                                       /* sx.val.l = constant                             */
1167
1168                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1169                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1170                         M_INTMOVE(s1, d);
1171                         if (IS_IMM32(iptr->sx.val.l))
1172                                 M_LXOR_IMM(iptr->sx.val.l, d);
1173                         else {
1174                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1175                                 M_LXOR(REG_ITMP2, d);
1176                         }
1177                         emit_store_dst(jd, iptr, d);
1178                         break;
1179
1180
1181                 /* floating operations ************************************************/
1182
1183                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1184
1185                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1186                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1187                         disp = dseg_add_s4(cd, 0x80000000);
1188                         M_FLTMOVE(s1, d);
1189                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1190                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1191                         emit_store_dst(jd, iptr, d);
1192                         break;
1193
1194                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1195
1196                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1197                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1198                         disp = dseg_add_s8(cd, 0x8000000000000000);
1199                         M_FLTMOVE(s1, d);
1200                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1201                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1202                         emit_store_dst(jd, iptr, d);
1203                         break;
1204
1205                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1206
1207                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1208                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1209                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1210                         if (s2 == d)
1211                                 M_FADD(s1, d);
1212                         else {
1213                                 M_FLTMOVE(s1, d);
1214                                 M_FADD(s2, d);
1215                         }
1216                         emit_store_dst(jd, iptr, d);
1217                         break;
1218
1219                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1220
1221                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1222                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1223                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1224                         if (s2 == d)
1225                                 M_DADD(s1, d);
1226                         else {
1227                                 M_FLTMOVE(s1, d);
1228                                 M_DADD(s2, d);
1229                         }
1230                         emit_store_dst(jd, iptr, d);
1231                         break;
1232
1233                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1234
1235                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1236                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1237                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1238                         if (s2 == d) {
1239                                 M_FLTMOVE(s2, REG_FTMP2);
1240                                 s2 = REG_FTMP2;
1241                         }
1242                         M_FLTMOVE(s1, d);
1243                         M_FSUB(s2, d);
1244                         emit_store_dst(jd, iptr, d);
1245                         break;
1246
1247                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1248
1249                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1250                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1251                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1252                         if (s2 == d) {
1253                                 M_FLTMOVE(s2, REG_FTMP2);
1254                                 s2 = REG_FTMP2;
1255                         }
1256                         M_FLTMOVE(s1, d);
1257                         M_DSUB(s2, d);
1258                         emit_store_dst(jd, iptr, d);
1259                         break;
1260
1261                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1262
1263                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1264                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1265                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1266                         if (s2 == d)
1267                                 M_FMUL(s1, d);
1268                         else {
1269                                 M_FLTMOVE(s1, d);
1270                                 M_FMUL(s2, d);
1271                         }
1272                         emit_store_dst(jd, iptr, d);
1273                         break;
1274
1275                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1276
1277                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1278                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1279                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1280                         if (s2 == d)
1281                                 M_DMUL(s1, d);
1282                         else {
1283                                 M_FLTMOVE(s1, d);
1284                                 M_DMUL(s2, d);
1285                         }
1286                         emit_store_dst(jd, iptr, d);
1287                         break;
1288
1289                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1290
1291                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1292                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1293                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1294                         if (s2 == d) {
1295                                 M_FLTMOVE(s2, REG_FTMP2);
1296                                 s2 = REG_FTMP2;
1297                         }
1298                         M_FLTMOVE(s1, d);
1299                         M_FDIV(s2, d);
1300                         emit_store_dst(jd, iptr, d);
1301                         break;
1302
1303                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1304
1305                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1306                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1307                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1308                         if (s2 == d) {
1309                                 M_FLTMOVE(s2, REG_FTMP2);
1310                                 s2 = REG_FTMP2;
1311                         }
1312                         M_FLTMOVE(s1, d);
1313                         M_DDIV(s2, d);
1314                         emit_store_dst(jd, iptr, d);
1315                         break;
1316
1317                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1318
1319                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1320                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1321                         M_CVTIF(s1, d);
1322                         emit_store_dst(jd, iptr, d);
1323                         break;
1324
1325                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1326
1327                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1328                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1329                         M_CVTID(s1, d);
1330                         emit_store_dst(jd, iptr, d);
1331                         break;
1332
1333                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1334
1335                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1336                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1337                         M_CVTLF(s1, d);
1338                         emit_store_dst(jd, iptr, d);
1339                         break;
1340
1341                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1342
1343                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1344                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1345                         M_CVTLD(s1, d);
1346                         emit_store_dst(jd, iptr, d);
1347                         break;
1348
1349                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1350
1351                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1352                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1353                         M_CVTFI(s1, d);
1354                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1355                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1356                                 ((REG_RESULT == d) ? 0 : 3);
1357                         M_BNE(disp);
1358                         M_FLTMOVE(s1, REG_FTMP1);
1359                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1360                         M_CALL(REG_ITMP2);
1361                         M_INTMOVE(REG_RESULT, d);
1362                         emit_store_dst(jd, iptr, d);
1363                         break;
1364
1365                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1366
1367                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1368                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1369                         M_CVTDI(s1, d);
1370                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1371                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1372                                 ((REG_RESULT == d) ? 0 : 3);
1373                         M_BNE(disp);
1374                         M_FLTMOVE(s1, REG_FTMP1);
1375                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1376                         M_CALL(REG_ITMP2);
1377                         M_INTMOVE(REG_RESULT, d);
1378                         emit_store_dst(jd, iptr, d);
1379                         break;
1380
1381                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1382
1383                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1384                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1385                         M_CVTFL(s1, d);
1386                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1387                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1388                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1389                                 ((REG_RESULT == d) ? 0 : 3);
1390                         M_BNE(disp);
1391                         M_FLTMOVE(s1, REG_FTMP1);
1392                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1393                         M_CALL(REG_ITMP2);
1394                         M_INTMOVE(REG_RESULT, d);
1395                         emit_store_dst(jd, iptr, d);
1396                         break;
1397
1398                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1399
1400                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1401                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1402                         M_CVTDL(s1, d);
1403                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1404                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1405                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1406                                 ((REG_RESULT == d) ? 0 : 3);
1407                         M_BNE(disp);
1408                         M_FLTMOVE(s1, REG_FTMP1);
1409                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1410                         M_CALL(REG_ITMP2);
1411                         M_INTMOVE(REG_RESULT, d);
1412                         emit_store_dst(jd, iptr, d);
1413                         break;
1414
1415                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1416
1417                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1418                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1419                         M_CVTFD(s1, d);
1420                         emit_store_dst(jd, iptr, d);
1421                         break;
1422
1423                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1424
1425                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1426                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1427                         M_CVTDF(s1, d);
1428                         emit_store_dst(jd, iptr, d);
1429                         break;
1430
1431                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1432                                           /* == => 0, < => 1, > => -1 */
1433
1434                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1435                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1436                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1437                         M_CLR(d);
1438                         M_MOV_IMM(1, REG_ITMP1);
1439                         M_MOV_IMM(-1, REG_ITMP2);
1440                         emit_ucomiss_reg_reg(cd, s1, s2);
1441                         M_CMOVULT(REG_ITMP1, d);
1442                         M_CMOVUGT(REG_ITMP2, d);
1443                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1444                         emit_store_dst(jd, iptr, d);
1445                         break;
1446
1447                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1448                                           /* == => 0, < => 1, > => -1 */
1449
1450                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1451                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1452                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1453                         M_CLR(d);
1454                         M_MOV_IMM(1, REG_ITMP1);
1455                         M_MOV_IMM(-1, REG_ITMP2);
1456                         emit_ucomiss_reg_reg(cd, s1, s2);
1457                         M_CMOVULT(REG_ITMP1, d);
1458                         M_CMOVUGT(REG_ITMP2, d);
1459                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1460                         emit_store_dst(jd, iptr, d);
1461                         break;
1462
1463                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1464                                           /* == => 0, < => 1, > => -1 */
1465
1466                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1467                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1468                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1469                         M_CLR(d);
1470                         M_MOV_IMM(1, REG_ITMP1);
1471                         M_MOV_IMM(-1, REG_ITMP2);
1472                         emit_ucomisd_reg_reg(cd, s1, s2);
1473                         M_CMOVULT(REG_ITMP1, d);
1474                         M_CMOVUGT(REG_ITMP2, d);
1475                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1476                         emit_store_dst(jd, iptr, d);
1477                         break;
1478
1479                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1480                                           /* == => 0, < => 1, > => -1 */
1481
1482                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1483                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1484                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1485                         M_CLR(d);
1486                         M_MOV_IMM(1, REG_ITMP1);
1487                         M_MOV_IMM(-1, REG_ITMP2);
1488                         emit_ucomisd_reg_reg(cd, s1, s2);
1489                         M_CMOVULT(REG_ITMP1, d);
1490                         M_CMOVUGT(REG_ITMP2, d);
1491                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1492                         emit_store_dst(jd, iptr, d);
1493                         break;
1494
1495
1496                 /* memory operations **************************************************/
1497
1498                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1499
1500                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1501                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1502                         /* implicit null-pointer check */
1503                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1504                         emit_store_dst(jd, iptr, d);
1505                         break;
1506
1507                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1508
1509                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1510                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1511                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1512                         /* implicit null-pointer check */
1513                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1514                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1515                         emit_store_dst(jd, iptr, d);
1516                         break;
1517
1518                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1519
1520                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1521                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1522                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1523                         /* implicit null-pointer check */
1524                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1525                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1526                         emit_store_dst(jd, iptr, d);
1527                         break;
1528
1529                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1530
1531                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1532                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1533                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1534                         /* implicit null-pointer check */
1535                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1536                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1537                         emit_store_dst(jd, iptr, d);
1538                         break;
1539
1540                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1541
1542                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1543                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1544                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1545                         /* implicit null-pointer check */
1546                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1547                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1548                         emit_store_dst(jd, iptr, d);
1549                         break;
1550
1551                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1552
1553                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1554                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1555                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1556                         /* implicit null-pointer check */
1557                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1558                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1559                         emit_store_dst(jd, iptr, d);
1560                         break;
1561
1562                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1563
1564                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1565                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1566                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1567                         /* implicit null-pointer check */
1568                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1569                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1570                         emit_store_dst(jd, iptr, d);
1571                         break;
1572
1573                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1574
1575                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1576                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1577                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1578                         /* implicit null-pointer check */
1579                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1580                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1581                         emit_store_dst(jd, iptr, d);
1582                         break;
1583
1584                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1585
1586                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1587                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1588                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1589                         /* implicit null-pointer check */
1590                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1591                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1592                         emit_store_dst(jd, iptr, d);
1593                         break;
1594
1595
1596                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1597
1598                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1599                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1600                         /* implicit null-pointer check */
1601                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1602                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1603                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1604                         break;
1605
1606                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1607
1608                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1609                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1610                         /* implicit null-pointer check */
1611                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1612                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1613                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1614                         break;
1615
1616                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1617
1618                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1619                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1620                         /* implicit null-pointer check */
1621                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1622                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1623                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1624                         break;
1625
1626                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1627
1628                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1629                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1630                         /* implicit null-pointer check */
1631                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1632                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1633                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
1634                         break;
1635
1636                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1637
1638                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1639                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1640                         /* implicit null-pointer check */
1641                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1642                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1643                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
1644                         break;
1645
1646                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1647
1648                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1649                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1650                         /* implicit null-pointer check */
1651                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1652                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1653                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
1654                         break;
1655
1656                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1657
1658                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1659                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1660                         /* implicit null-pointer check */
1661                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1662                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1663                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
1664                         break;
1665
1666                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1667
1668                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1669                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1670                         /* implicit null-pointer check */
1671                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1672                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1673
1674                         M_MOV(s1, REG_A0);
1675                         M_MOV(s3, REG_A1);
1676                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
1677                         M_CALL(REG_ITMP1);
1678                         emit_exception_check(cd, iptr);
1679
1680                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1681                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1682                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1683                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1684                         break;
1685
1686
1687                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1688
1689                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1690                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1691                         /* implicit null-pointer check */
1692                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1693                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1694                         break;
1695
1696                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1697
1698                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1699                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1700                         /* implicit null-pointer check */
1701                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1702                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray, data[0]), s1, s2, 1);
1703                         break;
1704
1705                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1706
1707                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1708                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1709                         /* implicit null-pointer check */
1710                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1711                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1712                         break;
1713
1714                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1715
1716                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1717                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1718                         /* implicit null-pointer check */
1719                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1720                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray, data[0]), s1, s2, 2);
1721                         break;
1722
1723                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1724
1725                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1726                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1727                         /* implicit null-pointer check */
1728                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1729
1730                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1731                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1732                         }
1733                         else {
1734                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1735                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
1736                         }
1737                         break;
1738
1739                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1740
1741                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1742                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1743                         /* implicit null-pointer check */
1744                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1745                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1746                         break;
1747
1748
1749                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1750
1751                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1752                                 uf        = iptr->sx.s23.s3.uf;
1753                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1754                                 disp      = dseg_add_unique_address(cd, NULL);
1755                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1756
1757                                 /* must be calculated before codegen_add_patch_ref */
1758
1759                                 if (opt_shownops)
1760                                         disp -= PATCHER_CALL_SIZE;
1761
1762 /*                              PROFILE_CYCLE_STOP; */
1763
1764                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1765
1766 /*                              PROFILE_CYCLE_START; */
1767                         }
1768                         else {
1769                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1770                                 fieldtype = fi->type;
1771                                 disp      = dseg_add_address(cd, &(fi->value));
1772                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1773
1774                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1775                                         PROFILE_CYCLE_STOP;
1776
1777                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1778
1779                                         PROFILE_CYCLE_START;
1780
1781                                         if (opt_shownops)
1782                                                 disp -= PATCHER_CALL_SIZE;
1783                                 }
1784                         }
1785
1786                         /* This approach is much faster than moving the field
1787                            address inline into a register. */
1788
1789                         M_ALD(REG_ITMP1, RIP, disp);
1790
1791                         switch (fieldtype) {
1792                         case TYPE_INT:
1793                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1794                                 M_ILD(d, REG_ITMP1, 0);
1795                                 break;
1796                         case TYPE_LNG:
1797                         case TYPE_ADR:
1798                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1799                                 M_LLD(d, REG_ITMP1, 0);
1800                                 break;
1801                         case TYPE_FLT:
1802                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1803                                 M_FLD(d, REG_ITMP1, 0);
1804                                 break;
1805                         case TYPE_DBL:
1806                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1807                                 M_DLD(d, REG_ITMP1, 0);
1808                                 break;
1809                         }
1810                         emit_store_dst(jd, iptr, d);
1811                         break;
1812
1813                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1814
1815                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1816                                 uf        = iptr->sx.s23.s3.uf;
1817                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1818                                 disp      = dseg_add_unique_address(cd, NULL);
1819                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1820
1821                                 /* must be calculated before codegen_add_patch_ref */
1822
1823                                 if (opt_shownops)
1824                                         disp -= PATCHER_CALL_SIZE;
1825
1826 /*                              PROFILE_CYCLE_STOP; */
1827
1828                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1829
1830 /*                              PROFILE_CYCLE_START; */
1831                         }
1832                         else {
1833                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1834                                 fieldtype = fi->type;
1835                                 disp      = dseg_add_address(cd, &(fi->value));
1836                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1837
1838                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1839                                         PROFILE_CYCLE_STOP;
1840
1841                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1842
1843                                         PROFILE_CYCLE_START;
1844
1845                                         if (opt_shownops)
1846                                                 disp -= PATCHER_CALL_SIZE;
1847                                 }
1848                         }
1849
1850                         /* This approach is much faster than moving the field
1851                            address inline into a register. */
1852
1853                         M_ALD(REG_ITMP1, RIP, disp);
1854
1855                         switch (fieldtype) {
1856                         case TYPE_INT:
1857                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1858                                 M_IST(s1, REG_ITMP1, 0);
1859                                 break;
1860                         case TYPE_LNG:
1861                         case TYPE_ADR:
1862                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1863                                 M_LST(s1, REG_ITMP1, 0);
1864                                 break;
1865                         case TYPE_FLT:
1866                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1867                                 M_FST(s1, REG_ITMP1, 0);
1868                                 break;
1869                         case TYPE_DBL:
1870                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1871                                 M_DST(s1, REG_ITMP1, 0);
1872                                 break;
1873                         }
1874                         break;
1875
1876                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1877                                           /* val = value (in current instruction)     */
1878                                           /* following NOP)                           */
1879
1880                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1881                                 uf        = iptr->sx.s23.s3.uf;
1882                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1883                                 disp      = dseg_add_unique_address(cd, NULL);
1884                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1885
1886                                 /* must be calculated before codegen_add_patch_ref */
1887
1888                                 if (opt_shownops)
1889                                         disp -= PATCHER_CALL_SIZE;
1890
1891 /*                              PROFILE_CYCLE_STOP; */
1892
1893                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1894
1895 /*                              PROFILE_CYCLE_START; */
1896                         }
1897                         else {
1898                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1899                                 fieldtype = fi->type;
1900                                 disp      = dseg_add_address(cd, &(fi->value));
1901                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1902
1903                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1904                                         PROFILE_CYCLE_STOP;
1905
1906                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1907
1908                                         PROFILE_CYCLE_START;
1909
1910                                         if (opt_shownops)
1911                                                 disp -= PATCHER_CALL_SIZE;
1912                                 }
1913                         }
1914
1915                         /* This approach is much faster than moving the field
1916                            address inline into a register. */
1917
1918                         M_ALD(REG_ITMP1, RIP, disp);
1919
1920                         switch (fieldtype) {
1921                         case TYPE_INT:
1922                         case TYPE_FLT:
1923                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1924                                 break;
1925                         case TYPE_LNG:
1926                         case TYPE_ADR:
1927                         case TYPE_DBL:
1928                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1929                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1930                                 else {
1931                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1932                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1933                                 }
1934                                 break;
1935                         }
1936                         break;
1937
1938                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1939
1940                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1941
1942                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1943                                 uf        = iptr->sx.s23.s3.uf;
1944                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1945                                 disp      = 0;
1946
1947 /*                              PROFILE_CYCLE_STOP; */
1948
1949                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1950
1951 /*                              PROFILE_CYCLE_START; */
1952                         }
1953                         else {
1954                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1955                                 fieldtype = fi->type;
1956                                 disp      = fi->offset;
1957                         }
1958
1959                         /* implicit null-pointer check */
1960                         switch (fieldtype) {
1961                         case TYPE_INT:
1962                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1963                                 M_ILD32(d, s1, disp);
1964                                 break;
1965                         case TYPE_LNG:
1966                         case TYPE_ADR:
1967                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1968                                 M_LLD32(d, s1, disp);
1969                                 break;
1970                         case TYPE_FLT:
1971                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1972                                 M_FLD32(d, s1, disp);
1973                                 break;
1974                         case TYPE_DBL:
1975                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1976                                 M_DLD32(d, s1, disp);
1977                                 break;
1978                         }
1979                         emit_store_dst(jd, iptr, d);
1980                         break;
1981
1982                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1983
1984                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1985                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1986
1987                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1988                                 uf        = iptr->sx.s23.s3.uf;
1989                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1990                                 disp      = 0;
1991
1992 /*                              PROFILE_CYCLE_STOP; */
1993
1994                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1995
1996 /*                              PROFILE_CYCLE_START; */
1997                         }
1998                         else {
1999                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2000                                 fieldtype = fi->type;
2001                                 disp      = fi->offset;
2002                         }
2003
2004                         /* implicit null-pointer check */
2005                         switch (fieldtype) {
2006                         case TYPE_INT:
2007                                 M_IST32(s2, s1, disp);
2008                                 break;
2009                         case TYPE_LNG:
2010                         case TYPE_ADR:
2011                                 M_LST32(s2, s1, disp);
2012                                 break;
2013                         case TYPE_FLT:
2014                                 M_FST32(s2, s1, disp);
2015                                 break;
2016                         case TYPE_DBL:
2017                                 M_DST32(s2, s1, disp);
2018                                 break;
2019                         }
2020                         break;
2021
2022                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2023                                           /* val = value (in current instruction)     */
2024                                           /* following NOP)                           */
2025
2026                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2027
2028                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2029                                 uf        = iptr->sx.s23.s3.uf;
2030                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2031                                 disp      = 0;
2032
2033 /*                              PROFILE_CYCLE_STOP; */
2034
2035                                 codegen_add_patch_ref(cd, PATCHER_putfieldconst, uf, 0);
2036
2037 /*                              PROFILE_CYCLE_START; */
2038                         }
2039                         else {
2040                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2041                                 fieldtype = fi->type;
2042                                 disp      = fi->offset;
2043                         }
2044
2045                         /* implicit null-pointer check */
2046                         switch (fieldtype) {
2047                         case TYPE_INT:
2048                         case TYPE_FLT:
2049                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2050                                 break;
2051                         case TYPE_LNG:
2052                         case TYPE_ADR:
2053                         case TYPE_DBL:
2054                                 /* XXX why no check for IS_IMM32? */
2055                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2056                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2057                                 break;
2058                         }
2059                         break;
2060
2061
2062                 /* branch operations **************************************************/
2063
2064                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2065
2066                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2067                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2068
2069                         PROFILE_CYCLE_STOP;
2070
2071 #ifdef ENABLE_VERIFIER
2072                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2073                                 uc = iptr->sx.s23.s2.uc;
2074
2075                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2076                         }
2077 #endif /* ENABLE_VERIFIER */
2078
2079                         M_CALL_IMM(0);                            /* passing exception pc */
2080                         M_POP(REG_ITMP2_XPC);
2081
2082                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2083                         M_JMP(REG_ITMP3);
2084                         break;
2085
2086                 case ICMD_GOTO:         /* ... ==> ...                                */
2087                 case ICMD_RET:
2088
2089                         emit_br(cd, iptr->dst.block);
2090                         ALIGNCODENOP;
2091                         break;
2092
2093                 case ICMD_JSR:          /* ... ==> ...                                */
2094
2095                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2096                         ALIGNCODENOP;
2097                         break;
2098
2099                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2100                 case ICMD_IFNONNULL:
2101
2102                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2103                         M_TEST(s1);
2104                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2105                         break;
2106
2107                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2108                 case ICMD_IFLT:
2109                 case ICMD_IFLE:
2110                 case ICMD_IFNE:
2111                 case ICMD_IFGT:
2112                 case ICMD_IFGE:
2113
2114                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2115                         M_ICMP_IMM(iptr->sx.val.i, s1);
2116                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2117                         break;
2118
2119                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2120                 case ICMD_IF_LNE:
2121                 case ICMD_IF_LLT:
2122                 case ICMD_IF_LGE:
2123                 case ICMD_IF_LGT:
2124                 case ICMD_IF_LLE:
2125
2126                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2127                         if (IS_IMM32(iptr->sx.val.l))
2128                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2129                         else {
2130                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2131                                 M_LCMP(REG_ITMP2, s1);
2132                         }
2133                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2134                         break;
2135
2136                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2137                 case ICMD_IF_ICMPNE:
2138                 case ICMD_IF_ICMPLT:
2139                 case ICMD_IF_ICMPGE:
2140                 case ICMD_IF_ICMPGT:
2141                 case ICMD_IF_ICMPLE:
2142
2143                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2144                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2145                         M_ICMP(s2, s1);
2146                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2147                         break;
2148
2149                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2150                 case ICMD_IF_ACMPNE:
2151
2152                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2153                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2154                         M_LCMP(s2, s1);
2155                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2156                         break;
2157
2158                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2159                 case ICMD_IF_LCMPNE:
2160                 case ICMD_IF_LCMPLT:
2161                 case ICMD_IF_LCMPGE:
2162                 case ICMD_IF_LCMPGT:
2163                 case ICMD_IF_LCMPLE:
2164
2165                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2166                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2167                         M_LCMP(s2, s1);
2168                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2169                         break;
2170
2171                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2172                 case ICMD_LRETURN:
2173
2174                         REPLACEMENT_POINT_RETURN(cd, iptr);
2175                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2176                         M_INTMOVE(s1, REG_RESULT);
2177                         goto nowperformreturn;
2178
2179                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2180
2181                         REPLACEMENT_POINT_RETURN(cd, iptr);
2182                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2183                         M_INTMOVE(s1, REG_RESULT);
2184
2185 #ifdef ENABLE_VERIFIER
2186                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2187                                 uc = iptr->sx.s23.s2.uc;
2188
2189                                 PROFILE_CYCLE_STOP;
2190
2191                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2192
2193                                 PROFILE_CYCLE_START;
2194                         }
2195 #endif /* ENABLE_VERIFIER */
2196                         goto nowperformreturn;
2197
2198                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2199                 case ICMD_DRETURN:
2200
2201                         REPLACEMENT_POINT_RETURN(cd, iptr);
2202                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2203                         M_FLTMOVE(s1, REG_FRESULT);
2204                         goto nowperformreturn;
2205
2206                 case ICMD_RETURN:      /* ...  ==> ...                                */
2207
2208                         REPLACEMENT_POINT_RETURN(cd, iptr);
2209
2210 nowperformreturn:
2211                         {
2212                         s4 i, p;
2213
2214                         p = cd->stackframesize;
2215
2216 #if !defined(NDEBUG)
2217                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2218                                 emit_verbosecall_exit(jd);
2219 #endif /* !defined(NDEBUG) */
2220
2221 #if defined(ENABLE_THREADS)
2222                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2223                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2224
2225                                 /* we need to save the proper return value */
2226                                 switch (iptr->opc) {
2227                                 case ICMD_IRETURN:
2228                                 case ICMD_ARETURN:
2229                                 case ICMD_LRETURN:
2230                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2231                                         break;
2232                                 case ICMD_FRETURN:
2233                                 case ICMD_DRETURN:
2234                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2235                                         break;
2236                                 }
2237
2238                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2239                                 M_CALL(REG_ITMP1);
2240
2241                                 /* and now restore the proper return value */
2242                                 switch (iptr->opc) {
2243                                 case ICMD_IRETURN:
2244                                 case ICMD_ARETURN:
2245                                 case ICMD_LRETURN:
2246                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2247                                         break;
2248                                 case ICMD_FRETURN:
2249                                 case ICMD_DRETURN:
2250                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2251                                         break;
2252                                 }
2253                         }
2254 #endif
2255
2256                         /* restore saved registers */
2257
2258                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2259                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2260                         }
2261                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2262                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2263                         }
2264
2265                         /* deallocate stack */
2266
2267                         if (cd->stackframesize)
2268                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2269
2270                         /* generate method profiling code */
2271
2272                         PROFILE_CYCLE_STOP;
2273
2274                         M_RET;
2275                         }
2276                         break;
2277
2278
2279                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2280                         {
2281                                 s4 i, l;
2282                                 branch_target_t *table;
2283
2284                                 table = iptr->dst.table;
2285
2286                                 l = iptr->sx.s23.s2.tablelow;
2287                                 i = iptr->sx.s23.s3.tablehigh;
2288
2289                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2290                                 M_INTMOVE(s1, REG_ITMP1);
2291
2292                                 if (l != 0)
2293                                         M_ISUB_IMM(l, REG_ITMP1);
2294
2295                                 /* number of targets */
2296                                 i = i - l + 1;
2297
2298                 /* range check */
2299
2300                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2301                                 emit_bugt(cd, table[0].block);
2302
2303                                 /* build jump table top down and use address of lowest entry */
2304
2305                                 table += i;
2306
2307                                 while (--i >= 0) {
2308                                         dseg_add_target(cd, table->block);
2309                                         --table;
2310                                 }
2311
2312                                 /* length of dataseg after last dseg_add_target is used
2313                                    by load */
2314
2315                                 M_MOV_IMM(0, REG_ITMP2);
2316                                 dseg_adddata(cd);
2317                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2318                                 M_JMP(REG_ITMP1);
2319                         }
2320                         break;
2321
2322
2323                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2324                         {
2325                                 s4 i;
2326                                 lookup_target_t *lookup;
2327
2328                                 lookup = iptr->dst.lookup;
2329
2330                                 i = iptr->sx.s23.s2.lookupcount;
2331
2332                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2333                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2334
2335                                 while (--i >= 0) {
2336                                         M_ICMP_IMM(lookup->value, s1);
2337                                         emit_beq(cd, lookup->target.block);
2338                                         lookup++;
2339                                 }
2340
2341                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2342                                 ALIGNCODENOP;
2343                         }
2344                         break;
2345
2346
2347                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2348
2349                         bte = iptr->sx.s23.s3.bte;
2350                         md  = bte->md;
2351                         goto gen_method;
2352
2353                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2354
2355                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2356                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2357                 case ICMD_INVOKEINTERFACE:
2358
2359                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2360
2361                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2362                                 lm = NULL;
2363                                 um = iptr->sx.s23.s3.um;
2364                                 md = um->methodref->parseddesc.md;
2365                         }
2366                         else {
2367                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2368                                 um = NULL;
2369                                 md = lm->parseddesc;
2370                         }
2371
2372 gen_method:
2373                         s3 = md->paramcount;
2374
2375                         MCODECHECK((20 * s3) + 128);
2376
2377                         /* copy arguments to registers or stack location */
2378
2379                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2380                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2381                                 d   = md->params[s3].regoff;
2382
2383                                 /* already preallocated (ARGVAR)? */
2384
2385                                 if (var->flags & PREALLOC)
2386                                         continue;
2387
2388                                 if (IS_INT_LNG_TYPE(var->type)) {
2389                                         if (!md->params[s3].inmemory) {
2390                                                 s1 = emit_load(jd, iptr, var, d);
2391                                                 M_INTMOVE(s1, d);
2392                                         }
2393                                         else {
2394                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2395                                                 M_LST(s1, REG_SP, d * 8);
2396                                         }
2397                                 }
2398                                 else {
2399                                         if (!md->params[s3].inmemory) {
2400                                                 s1 = emit_load(jd, iptr, var, d);
2401                                                 M_FLTMOVE(s1, d);
2402                                         }
2403                                         else {
2404                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2405
2406                                                 if (IS_2_WORD_TYPE(var->type))
2407                                                         M_DST(s1, REG_SP, d * 8);
2408                                                 else
2409                                                         M_FST(s1, REG_SP, d * 8);
2410                                         }
2411                                 }
2412                         }
2413
2414                         /* generate method profiling code */
2415
2416                         PROFILE_CYCLE_STOP;
2417
2418                         switch (iptr->opc) {
2419                         case ICMD_BUILTIN:
2420                                 M_MOV_IMM(bte->fp, REG_ITMP1);
2421                                 M_CALL(REG_ITMP1);
2422
2423                                 emit_exception_check(cd, iptr);
2424                                 break;
2425
2426                         case ICMD_INVOKESPECIAL:
2427                                 emit_nullpointer_check(cd, iptr, REG_A0);
2428                                 /* fall through */
2429
2430                         case ICMD_INVOKESTATIC:
2431                                 if (lm == NULL) {
2432                                         disp = dseg_add_unique_address(cd, NULL);
2433                                         disp = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
2434
2435                                         /* must be calculated before codegen_add_patch_ref */
2436
2437                                         if (opt_shownops)
2438                                                 disp -= PATCHER_CALL_SIZE;
2439
2440                                         codegen_add_patch_ref(cd, PATCHER_invokestatic_special,
2441                                                                                   um, disp);
2442
2443 /*                                      a = 0; */
2444                                 }
2445                                 else {
2446                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2447                                         disp = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
2448
2449 /*                                      a = (ptrint) lm->stubroutine; */
2450                                 }
2451
2452 /*                              M_MOV_IMM(a, REG_ITMP2); */
2453                                 M_ALD(REG_ITMP2, RIP, disp);
2454                                 M_CALL(REG_ITMP2);
2455                                 break;
2456
2457                         case ICMD_INVOKEVIRTUAL:
2458                                 if (lm == NULL) {
2459                                         codegen_add_patch_ref(cd, PATCHER_invokevirtual, um, 0);
2460
2461                                         s1 = 0;
2462                                 }
2463                                 else {
2464                                         s1 = OFFSET(vftbl_t, table[0]) +
2465                                                 sizeof(methodptr) * lm->vftblindex;
2466                                 }
2467
2468                                 /* implicit null-pointer check */
2469                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_objectheader, vftbl));
2470                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2471                                 M_CALL(REG_ITMP3);
2472                                 break;
2473
2474                         case ICMD_INVOKEINTERFACE:
2475                                 if (lm == NULL) {
2476                                         codegen_add_patch_ref(cd, PATCHER_invokeinterface, um, 0);
2477
2478                                         s1 = 0;
2479                                         s2 = 0;
2480                                 }
2481                                 else {
2482                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2483                                                 sizeof(methodptr) * lm->class->index;
2484
2485                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2486                                 }
2487
2488                                 /* implicit null-pointer check */
2489                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_objectheader, vftbl));
2490                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2491                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2492                                 M_CALL(REG_ITMP3);
2493                                 break;
2494                         }
2495
2496                         /* generate method profiling code */
2497
2498                         PROFILE_CYCLE_START;
2499
2500                         /* store size of call code in replacement point */
2501
2502                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2503
2504                         /* store return value */
2505
2506                         switch (md->returntype.type) {
2507                         case TYPE_INT:
2508                         case TYPE_LNG:
2509                         case TYPE_ADR:
2510                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2511                                 M_INTMOVE(REG_RESULT, s1);
2512                                 emit_store_dst(jd, iptr, s1);
2513                                 break;
2514                         case TYPE_FLT:
2515                         case TYPE_DBL:
2516                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2517                                 M_FLTMOVE(REG_FRESULT, s1);
2518                                 emit_store_dst(jd, iptr, s1);
2519                                 break;
2520                         default:
2521                                 /* TYPE_VOID */
2522                                 break;
2523                         }
2524                         break;
2525
2526
2527                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2528
2529                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2530                                 /* object type cast-check */
2531
2532                                 classinfo *super;
2533                                 vftbl_t   *supervftbl;
2534                                 s4         superindex;
2535
2536                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2537                                         super      = NULL;
2538                                         superindex = 0;
2539                                         supervftbl = NULL;
2540                                 }
2541                                 else {
2542                                         super      = iptr->sx.s23.s3.c.cls;
2543                                         superindex = super->index;
2544                                         supervftbl = super->vftbl;
2545                                 }
2546
2547                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2548                                         CODEGEN_CRITICAL_SECTION_NEW;
2549
2550                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2551
2552                                 /* if class is not resolved, check which code to call */
2553
2554                                 if (super == NULL) {
2555                                         M_TEST(s1);
2556                                         emit_label_beq(cd, BRANCH_LABEL_1);
2557
2558                                         codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2559                                                                                   iptr->sx.s23.s3.c.ref, 0);
2560
2561                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2562                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2563                                         emit_label_beq(cd, BRANCH_LABEL_2);
2564                                 }
2565
2566                                 /* interface checkcast code */
2567
2568                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2569                                         if (super != NULL) {
2570                                                 M_TEST(s1);
2571                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2572                                         }
2573
2574                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2575
2576                                         if (super == NULL) {
2577                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_interface,
2578                                                                                           iptr->sx.s23.s3.c.ref,
2579                                                                                           0);
2580                                         }
2581
2582                                         M_ILD32(REG_ITMP3,
2583                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2584                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2585                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2586
2587                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2588                                                         OFFSET(vftbl_t, interfacetable[0]) -
2589                                                         superindex * sizeof(methodptr*));
2590                                         M_TEST(REG_ITMP3);
2591                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2592
2593                                         if (super == NULL)
2594                                                 emit_label_br(cd, BRANCH_LABEL_4);
2595                                         else
2596                                                 emit_label(cd, BRANCH_LABEL_3);
2597                                 }
2598
2599                                 /* class checkcast code */
2600
2601                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2602                                         if (super == NULL) {
2603                                                 emit_label(cd, BRANCH_LABEL_2);
2604                                         }
2605                                         else {
2606                                                 M_TEST(s1);
2607                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2608                                         }
2609
2610                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2611
2612                                         if (super == NULL) {
2613                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_class,
2614                                                                                           iptr->sx.s23.s3.c.ref,
2615                                                                                           0);
2616                                         }
2617
2618                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2619
2620                                         CODEGEN_CRITICAL_SECTION_START;
2621
2622                                         M_ILD32(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2623
2624                                         /*                                      if (s1 != REG_ITMP1) { */
2625                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2626                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2627                                         /*                                                                                              REG_ITMP1); */
2628                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2629                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2630                                         /*                                                                                              REG_ITMP3); */
2631                                         /*  #if defined(ENABLE_THREADS) */
2632                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2633                                         /*  #endif */
2634                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2635
2636                                         /*                                      } else { */
2637
2638                                         M_ILD32(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
2639                                         M_ISUB(REG_ITMP3, REG_ITMP2);
2640                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2641                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2642                                         /*                                      } */
2643
2644                                         CODEGEN_CRITICAL_SECTION_END;
2645
2646                                         M_ICMP(REG_ITMP3, REG_ITMP2);
2647                                         emit_classcast_check(cd, iptr, BRANCH_UGT, REG_ITMP3, s1);
2648
2649                                         if (super != NULL)
2650                                                 emit_label(cd, BRANCH_LABEL_5);
2651                                 }
2652
2653                                 if (super == NULL) {
2654                                         emit_label(cd, BRANCH_LABEL_1);
2655                                         emit_label(cd, BRANCH_LABEL_4);
2656                                 }
2657
2658                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2659                         }
2660                         else {
2661                                 /* array type cast-check */
2662
2663                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2664                                 M_INTMOVE(s1, REG_A0);
2665
2666                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2667                                         codegen_add_patch_ref(cd, PATCHER_builtin_arraycheckcast,
2668                                                                                   iptr->sx.s23.s3.c.ref, 0);
2669                                 }
2670
2671                                 M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2672                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2673                                 M_CALL(REG_ITMP1);
2674
2675                                 /* s1 may have been destroyed over the function call */
2676                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2677                                 M_TEST(REG_RESULT);
2678                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2679
2680                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2681                         }
2682
2683                         M_INTMOVE(s1, d);
2684                         emit_store_dst(jd, iptr, d);
2685                         break;
2686
2687                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2688
2689                         {
2690                         classinfo *super;
2691                         vftbl_t   *supervftbl;
2692                         s4         superindex;
2693
2694                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2695                                 super      = NULL;
2696                                 superindex = 0;
2697                                 supervftbl = NULL;
2698                         }
2699                         else {
2700                                 super      = iptr->sx.s23.s3.c.cls;
2701                                 superindex = super->index;
2702                                 supervftbl = super->vftbl;
2703                         }
2704
2705                         if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2706                                 CODEGEN_CRITICAL_SECTION_NEW;
2707
2708                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2709                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2710
2711                         if (s1 == d) {
2712                                 M_INTMOVE(s1, REG_ITMP1);
2713                                 s1 = REG_ITMP1;
2714                         }
2715
2716                         M_CLR(d);
2717
2718                         /* if class is not resolved, check which code to call */
2719
2720                         if (super == NULL) {
2721                                 M_TEST(s1);
2722                                 emit_label_beq(cd, BRANCH_LABEL_1);
2723
2724                                 codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2725                                                                           iptr->sx.s23.s3.c.ref, 0);
2726
2727                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2728                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2729                                 emit_label_beq(cd, BRANCH_LABEL_2);
2730                         }
2731
2732                         /* interface instanceof code */
2733
2734                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2735                                 if (super != NULL) {
2736                                         M_TEST(s1);
2737                                         emit_label_beq(cd, BRANCH_LABEL_3);
2738                                 }
2739
2740                                 M_ALD(REG_ITMP1, s1, OFFSET(java_objectheader, vftbl));
2741
2742                                 if (super == NULL) {
2743                                         codegen_add_patch_ref(cd, PATCHER_instanceof_interface,
2744                                                                                   iptr->sx.s23.s3.c.ref, 0);
2745                                 }
2746
2747                                 M_ILD32(REG_ITMP3,
2748                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2749                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2750
2751                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2752
2753                                 M_BLE(a);
2754                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2755                                                 OFFSET(vftbl_t, interfacetable[0]) -
2756                                                 superindex * sizeof(methodptr*));
2757                                 M_TEST(REG_ITMP1);
2758                                 M_SETNE(d);
2759
2760                                 if (super == NULL)
2761                                         emit_label_br(cd, BRANCH_LABEL_4);
2762                                 else
2763                                         emit_label(cd, BRANCH_LABEL_3);
2764                         }
2765
2766                         /* class instanceof code */
2767
2768                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2769                                 if (super == NULL) {
2770                                         emit_label(cd, BRANCH_LABEL_2);
2771                                 }
2772                                 else {
2773                                         M_TEST(s1);
2774                                         emit_label_beq(cd, BRANCH_LABEL_5);
2775                                 }
2776
2777                                 M_ALD(REG_ITMP1, s1, OFFSET(java_objectheader, vftbl));
2778
2779                                 if (super == NULL) {
2780                                         codegen_add_patch_ref(cd, PATCHER_instanceof_class,
2781                                                                                   iptr->sx.s23.s3.c.ref, 0);
2782                                 }
2783
2784                                 M_MOV_IMM(supervftbl, REG_ITMP2);
2785
2786                                 CODEGEN_CRITICAL_SECTION_START;
2787
2788                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
2789                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2790                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2791
2792                                 CODEGEN_CRITICAL_SECTION_END;
2793
2794                                 M_ISUB(REG_ITMP2, REG_ITMP1);
2795                                 M_CLR(d); /* may be REG_ITMP2 */
2796                                 M_ICMP(REG_ITMP3, REG_ITMP1);
2797                                 M_SETULE(d);
2798
2799                                 if (super != NULL)
2800                                         emit_label(cd, BRANCH_LABEL_5);
2801                         }
2802
2803                         if (super == NULL) {
2804                                 emit_label(cd, BRANCH_LABEL_1);
2805                                 emit_label(cd, BRANCH_LABEL_4);
2806                         }
2807
2808                         emit_store_dst(jd, iptr, d);
2809                         }
2810                         break;
2811
2812                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2813
2814                         /* check for negative sizes and copy sizes to stack if necessary  */
2815
2816                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2817
2818                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2819
2820                                 /* copy SAVEDVAR sizes to stack */
2821                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2822
2823                                 /* Already Preallocated? */
2824                                 if (!(var->flags & PREALLOC)) {
2825                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2826                                         M_LST(s2, REG_SP, s1 * 8);
2827                                 }
2828                         }
2829
2830                         /* is a patcher function set? */
2831
2832                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2833                                 codegen_add_patch_ref(cd, PATCHER_builtin_multianewarray,
2834                                                                           iptr->sx.s23.s3.c.ref, 0);
2835                         }
2836
2837                         /* a0 = dimension count */
2838
2839                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2840
2841                         /* a1 = classinfo */
2842
2843                         M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2844
2845                         /* a2 = pointer to dimensions = stack pointer */
2846
2847                         M_MOV(REG_SP, REG_A2);
2848
2849                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2850                         M_CALL(REG_ITMP1);
2851
2852                         /* check for exception before result assignment */
2853
2854                         emit_exception_check(cd, iptr);
2855
2856                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2857                         M_INTMOVE(REG_RESULT, s1);
2858                         emit_store_dst(jd, iptr, s1);
2859                         break;
2860
2861                 default:
2862                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2863                                                                                    iptr->opc);
2864                         return false;
2865         } /* switch */
2866
2867         } /* for instruction */
2868
2869         MCODECHECK(512); /* XXX require a lower number? */
2870
2871         /* At the end of a basic block we may have to append some nops,
2872            because the patcher stub calling code might be longer than the
2873            actual instruction. So codepatching does not change the
2874            following block unintentionally. */
2875
2876         if (cd->mcodeptr < cd->lastmcodeptr) {
2877                 while (cd->mcodeptr < cd->lastmcodeptr) {
2878                         M_NOP;
2879                 }
2880         }
2881
2882         } /* if (bptr -> flags >= BBREACHED) */
2883         } /* for basic block */
2884
2885         dseg_createlinenumbertable(cd);
2886
2887         /* generate stubs */
2888
2889         emit_patcher_stubs(jd);
2890         REPLACEMENT_EMIT_STUBS(jd);
2891
2892         /* everything's ok */
2893
2894         return true;
2895 }
2896
2897
2898 /* codegen_emit_stub_compiler **************************************************
2899
2900    Emit a stub routine which calls the compiler.
2901
2902 *******************************************************************************/
2903
2904 void codegen_emit_stub_compiler(jitdata *jd)
2905 {
2906         methodinfo  *m;
2907         codegendata *cd;
2908
2909         /* get required compiler data */
2910
2911         m  = jd->m;
2912         cd = jd->cd;
2913
2914         /* code for the stub */
2915
2916         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
2917         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
2918         M_JMP(REG_ITMP3);
2919 }
2920
2921
2922 /* codegen_emit_stub_native ****************************************************
2923
2924    Emits a stub routine which calls a native method.
2925
2926 *******************************************************************************/
2927
2928 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f)
2929 {
2930         methodinfo  *m;
2931         codeinfo    *code;
2932         codegendata *cd;
2933         methoddesc  *md;
2934         s4           nativeparams;
2935         s4           i, j;
2936         s4           t;
2937         s4           s1, s2;
2938
2939         /* get required compiler data */
2940
2941         m    = jd->m;
2942         code = jd->code;
2943         cd   = jd->cd;
2944
2945         /* initialize variables */
2946
2947         md = m->parseddesc;
2948         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
2949
2950         /* calculate stack frame size */
2951
2952         cd->stackframesize =
2953                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
2954                 sizeof(localref_table) / SIZEOF_VOID_P +
2955                 md->paramcount +
2956                 1 +                       /* functionptr, TODO: store in data segment */
2957                 nmd->memuse;
2958
2959         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
2960
2961         /* create method header */
2962
2963         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2964         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2965         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
2966         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2967         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2968         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2969         (void) dseg_addlinenumbertablesize(cd);
2970         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
2971
2972 #if defined(ENABLE_PROFILING)
2973         /* generate native method profiling code */
2974
2975         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
2976                 /* count frequency */
2977
2978                 M_MOV_IMM(code, REG_ITMP3);
2979                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
2980         }
2981 #endif
2982
2983         /* generate stub code */
2984
2985         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2986
2987 #if !defined(NDEBUG)
2988         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2989                 emit_verbosecall_enter(jd);
2990 #endif
2991
2992         /* get function address (this must happen before the stackframeinfo) */
2993
2994 #if !defined(WITH_STATIC_CLASSPATH)
2995         if (f == NULL)
2996                 codegen_add_patch_ref(cd, PATCHER_resolve_native, m, 0);
2997 #endif
2998
2999         M_MOV_IMM(f, REG_ITMP3);
3000
3001         /* save integer and float argument registers */
3002
3003         for (i = 0; i < md->paramcount; i++) {
3004                 if (!md->params[i].inmemory) {
3005                         s1 = md->params[i].regoff;
3006
3007                         switch (md->paramtypes[i].type) {
3008                         case TYPE_INT:
3009                         case TYPE_LNG:
3010                         case TYPE_ADR:
3011                                 M_LST(s1, REG_SP, i * 8);
3012                                 break;
3013                         case TYPE_FLT:
3014                         case TYPE_DBL:
3015                                 M_DST(s1, REG_SP, i * 8);
3016                                 break;
3017                         }
3018                 }
3019         }
3020
3021         M_AST(REG_ITMP3, REG_SP, md->paramcount * 8);
3022
3023         /* create dynamic stack info */
3024
3025         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3026         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3027         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3028         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3029         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3030         M_CALL(REG_ITMP1);
3031
3032         /* restore integer and float argument registers */
3033
3034         for (i = 0; i < md->paramcount; i++) {
3035                 if (!md->params[i].inmemory) {
3036                         s1 = md->params[i].regoff;
3037
3038                         switch (md->paramtypes[i].type) {
3039                         case TYPE_INT:
3040                         case TYPE_LNG:
3041                         case TYPE_ADR:
3042                                 M_LLD(s1, REG_SP, i * 8);
3043                                 break;
3044                         case TYPE_FLT:
3045                         case TYPE_DBL:
3046                                 M_DLD(s1, REG_SP, i * 8);
3047                                 break;
3048                         }
3049                 }
3050         }
3051
3052         M_ALD(REG_ITMP3, REG_SP, md->paramcount * 8);
3053
3054         /* copy or spill arguments to new locations */
3055
3056         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3057                 t  = md->paramtypes[i].type;
3058                 s2 = nmd->params[j].regoff;
3059
3060                 if (IS_INT_LNG_TYPE(t)) {
3061                         if (!md->params[i].inmemory) {
3062                                 s1 = md->params[i].regoff;
3063
3064                                 if (!nmd->params[j].inmemory)
3065                                         M_INTMOVE(s1, s2);
3066                                 else
3067                                         M_LST(s1, REG_SP, s2 * 8);
3068                         }
3069                         else {
3070                                 s1 = md->params[i].regoff + cd->stackframesize + 1;/* +1 (RA) */
3071                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
3072                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
3073                         }
3074                 }
3075                 else {
3076                         /* We only copy spilled float arguments, as the float
3077                            argument registers keep unchanged. */
3078
3079                         if (md->params[i].inmemory) {
3080                                 s1 = md->params[i].regoff + cd->stackframesize + 1;/* +1 (RA) */
3081
3082                                 if (IS_2_WORD_TYPE(t)) {
3083                                         M_DLD(REG_FTMP1, REG_SP, s1 * 8);
3084                                         M_DST(REG_FTMP1, REG_SP, s2 * 8);
3085                                 }
3086                                 else {
3087                                         M_FLD(REG_FTMP1, REG_SP, s1 * 8);
3088                                         M_FST(REG_FTMP1, REG_SP, s2 * 8);
3089                                 }
3090                         }
3091                 }
3092         }
3093
3094         /* put class into second argument register */
3095
3096         if (m->flags & ACC_STATIC)
3097                 M_MOV_IMM(m->class, REG_A1);
3098
3099         /* put env into first argument register */
3100
3101         M_MOV_IMM(_Jv_env, REG_A0);
3102
3103         /* do the native function call */
3104
3105         M_CALL(REG_ITMP3);
3106
3107         /* save return value */
3108
3109         switch (md->returntype.type) {
3110         case TYPE_INT:
3111         case TYPE_LNG:
3112         case TYPE_ADR:
3113                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3114                 break;
3115         case TYPE_FLT:
3116         case TYPE_DBL:
3117                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3118                 break;
3119         case TYPE_VOID:
3120                 break;
3121         }
3122
3123 #if !defined(NDEBUG)
3124         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3125                 emit_verbosecall_exit(jd);
3126 #endif
3127
3128         /* remove native stackframe info */
3129
3130         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3131         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3132         M_CALL(REG_ITMP1);
3133         M_MOV(REG_RESULT, REG_ITMP3);
3134
3135         /* restore return value */
3136
3137         switch (md->returntype.type) {
3138         case TYPE_INT:
3139         case TYPE_LNG:
3140         case TYPE_ADR:
3141                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3142                 break;
3143         case TYPE_FLT:
3144         case TYPE_DBL:
3145                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3146                 break;
3147         case TYPE_VOID:
3148                 break;
3149         }
3150
3151         /* remove stackframe */
3152
3153         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3154
3155         /* test for exception */
3156
3157         M_TEST(REG_ITMP3);
3158         M_BNE(1);
3159         M_RET;
3160
3161         /* handle exception */
3162
3163         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3164         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3165         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3166
3167         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3168         M_JMP(REG_ITMP3);
3169
3170         /* generate patcher stubs */
3171
3172         emit_patcher_stubs(jd);
3173 }
3174
3175
3176 /*
3177  * These are local overrides for various environment variables in Emacs.
3178  * Please do not remove this and leave it at the end of the file, where
3179  * Emacs will automagically detect them.
3180  * ---------------------------------------------------------------------
3181  * Local variables:
3182  * mode: c
3183  * indent-tabs-mode: t
3184  * c-basic-offset: 4
3185  * tab-width: 4
3186  * End:
3187  * vim:noexpandtab:sw=4:ts=4:
3188  */