src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25 */
  26
  27
  28 #include "config.h"
  29
  30 #include <assert.h>
  31 #include <stdio.h>
  32
  33 #include "vm/types.h"
  34
  35 #include "md-abi.h"
  36
  37 #include "vm/jit/x86_64/arch.h"
  38 #include "vm/jit/x86_64/codegen.h"
  39 #include "vm/jit/x86_64/emit.h"
  40
  41 #include "mm/memory.h"
  42
  43 #include "native/jni.h"
  44 #include "native/localref.h"
  45 #include "native/native.h"
  46
  47 #include "threads/lock-common.h"
  48
  49 #include "vm/builtin.h"
  50 #include "vm/exceptions.h"
  51 #include "vm/global.h"
  52 #include "vm/stringlocal.h"
  53 #include "vm/vm.h"
  54
  55 #include "vm/jit/abi.h"
  56 #include "vm/jit/asmpart.h"
  57 #include "vm/jit/codegen-common.h"
  58 #include "vm/jit/dseg.h"
  59 #include "vm/jit/emit-common.h"
  60 #include "vm/jit/jit.h"
  61 #include "vm/jit/methodheader.h"
  62 #include "vm/jit/parse.h"
  63 #include "vm/jit/patcher-common.h"
  64 #include "vm/jit/reg.h"
  65 #include "vm/jit/replace.h"
  66 #include "vm/jit/stacktrace.h"
  67
  68 #if defined(ENABLE_LSRA)
  69 # include "vm/jit/allocator/lsra.h"
  70 #endif
  71
  72 #include "vmcore/loader.h"
  73 #include "vmcore/options.h"
  74 #include "vmcore/statistics.h"
  75
  76
  77 /* codegen_emit ****************************************************************
  78
  79    Generates machine code.
  80
  81 *******************************************************************************/
  82
  83 bool codegen_emit(jitdata *jd)
  84 {
  85         methodinfo         *m;
  86         codeinfo           *code;
  87         codegendata        *cd;
  88         registerdata       *rd;
  89         s4                  len, s1, s2, s3, d, disp;
  90         u2                  currentline;
  91         ptrint              a;
  92         varinfo            *var, *dst;
  93         basicblock         *bptr;
  94         instruction        *iptr;
  95         exception_entry    *ex;
  96         constant_classref  *cr;
  97         unresolved_class   *uc;
  98         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
  99         unresolved_method  *um;
 100         builtintable_entry *bte;
 101         methoddesc         *md;
 102         fieldinfo          *fi;
 103         unresolved_field   *uf;
 104         s4                  fieldtype;
 105         s4                 varindex;
 106
 107         /* get required compiler data */
 108
 109         m    = jd->m;
 110         code = jd->code;
 111         cd   = jd->cd;
 112         rd   = jd->rd;
 113
 114         /* prevent compiler warnings */
 115
 116         d   = 0;
 117         lm  = NULL;
 118         um  = NULL;
 119         bte = NULL;
 120
 121         {
 122         s4 i, p, t, l;
 123         s4 savedregs_num;
 124
 125         savedregs_num = 0;
 126
 127         /* space to save used callee saved registers */
 128
 129         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 130         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 131
 132         cd->stackframesize = rd->memuse + savedregs_num;
 133
 134 #if defined(ENABLE_THREADS)
 135         /* space to save argument of monitor_enter */
 136
 137         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 138                 cd->stackframesize++;
 139 #endif
 140
 141         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 142            native code e.g. libc or jni (alignment problems with
 143            movaps). */
 144
 145         if (!jd->isleafmethod || opt_verbosecall)
 146                 cd->stackframesize |= 0x1;
 147
 148         /* create method header */
 149
 150         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 151         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 152
 153 #if defined(ENABLE_THREADS)
 154         /* IsSync contains the offset relative to the stack pointer for the
 155            argument of monitor_exit used in the exception handler. Since the
 156            offset could be zero and give a wrong meaning of the flag it is
 157            offset by one.
 158         */
 159
 160         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 161                 (void) dseg_add_unique_s4(cd, (rd->memuse + 1) * 8); /* IsSync        */
 162         else
 163 #endif
 164                 (void) dseg_add_unique_s4(cd, 0);                  /* IsSync          */
 165
 166         (void) dseg_add_unique_s4(cd, jd->isleafmethod);               /* IsLeaf  */
 167         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 168         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 169
 170         (void) dseg_addlinenumbertablesize(cd);
 171
 172         (void) dseg_add_unique_s4(cd, jd->exceptiontablelength); /* ExTableSize   */
 173
 174         /* create exception table */
 175
 176         for (ex = jd->exceptiontable; ex != NULL; ex = ex->down) {
 177                 dseg_add_target(cd, ex->start);
 178                 dseg_add_target(cd, ex->end);
 179                 dseg_add_target(cd, ex->handler);
 180                 (void) dseg_add_unique_address(cd, ex->catchtype.any);
 181         }
 182
 183 #if defined(ENABLE_PROFILING)
 184         /* generate method profiling code */
 185
 186         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 187                 /* count frequency */
 188
 189                 M_MOV_IMM(code, REG_ITMP3);
 190                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 191
 192                 PROFILE_CYCLE_START;
 193         }
 194 #endif
 195
 196         /* create stack frame (if necessary) */
 197
 198         if (cd->stackframesize)
 199                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 200
 201         /* save used callee saved registers */
 202
 203         p = cd->stackframesize;
 204         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 205                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 206         }
 207         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 208                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 209         }
 210
 211         /* take arguments out of register or stack frame */
 212
 213         md = m->parseddesc;
 214
 215         for (p = 0, l = 0; p < md->paramcount; p++) {
 216                 t = md->paramtypes[p].type;
 217
 218                 varindex = jd->local_map[l * 5 + t];
 219
 220                 l++;
 221                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 222                         l++;
 223
 224                 if (varindex == UNUSED)
 225                         continue;
 226
 227                 var = VAR(varindex);
 228
 229                 s1 = md->params[p].regoff;
 230
 231                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 232                         if (!md->params[p].inmemory) {           /* register arguments    */
 233                                 if (!IS_INMEMORY(var->flags))
 234                                         M_INTMOVE(s1, var->vv.regoff);
 235                                 else
 236                                     M_LST(s1, REG_SP, var->vv.regoff);
 237                         }
 238                         else {                                 /* stack arguments       */
 239                                 if (!IS_INMEMORY(var->flags))
 240                                         /* + 8 for return address */
 241                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 242                                 else
 243                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 244                         }
 245                 }
 246                 else {                                     /* floating args         */
 247                         if (!md->params[p].inmemory) {           /* register arguments    */
 248                                 if (!IS_INMEMORY(var->flags))
 249                                         M_FLTMOVE(s1, var->vv.regoff);
 250                                 else
 251                                         M_DST(s1, REG_SP, var->vv.regoff);
 252                         }
 253                         else {                                 /* stack arguments       */
 254                                 if (!IS_INMEMORY(var->flags))
 255                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 256                                 else
 257                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 258                         }
 259                 }
 260         }
 261
 262         /* save monitorenter argument */
 263
 264 #if defined(ENABLE_THREADS)
 265         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 266                 /* stack offset for monitor argument */
 267
 268                 s1 = rd->memuse;
 269
 270                 if (opt_verbosecall) {
 271                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 272
 273                         for (p = 0; p < INT_ARG_CNT; p++)
 274                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 275
 276                         for (p = 0; p < FLT_ARG_CNT; p++)
 277                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 278
 279                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 280                 }
 281
 282                 /* decide which monitor enter function to call */
 283
 284                 if (m->flags & ACC_STATIC) {
 285                         M_MOV_IMM(&m->class->object.header, REG_A0);
 286                 }
 287                 else {
 288                         M_TEST(REG_A0);
 289                         M_BNE(8);
 290                         M_ALD_MEM(REG_A0, EXCEPTION_HARDWARE_NULLPOINTER);
 291                 }
 292
 293                 M_AST(REG_A0, REG_SP, s1 * 8);
 294                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 295                 M_CALL(REG_ITMP1);
 296
 297                 if (opt_verbosecall) {
 298                         for (p = 0; p < INT_ARG_CNT; p++)
 299                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 300
 301                         for (p = 0; p < FLT_ARG_CNT; p++)
 302                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 303
 304                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 305                 }
 306         }
 307 #endif
 308
 309 #if !defined(NDEBUG)
 310         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 311                 emit_verbosecall_enter(jd);
 312 #endif /* !defined(NDEBUG) */
 313
 314         }
 315
 316         /* end of header generation */
 317
 318         /* create replacement points */
 319
 320         REPLACEMENT_POINTS_INIT(cd, jd);
 321
 322         /* walk through all basic blocks */
 323
 324         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 325
 326                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 327
 328                 if (bptr->flags >= BBREACHED) {
 329
 330                 /* branch resolving */
 331
 332                 codegen_resolve_branchrefs(cd, bptr);
 333
 334                 /* handle replacement points */
 335
 336                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 337
 338                 /* copy interface registers to their destination */
 339
 340                 len = bptr->indepth;
 341                 MCODECHECK(512);
 342
 343 #if defined(ENABLE_PROFILING)
 344                 /* generate basicblock profiling code */
 345
 346                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 347                         /* count frequency */
 348
 349                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 350                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 351
 352                         /* if this is an exception handler, start profiling again */
 353
 354                         if (bptr->type == BBTYPE_EXH)
 355                                 PROFILE_CYCLE_START;
 356                 }
 357 #endif
 358
 359 #if defined(ENABLE_LSRA)
 360                 if (opt_lsra) {
 361                         while (len) {
 362                                 len--;
 363                                 src = bptr->invars[len];
 364                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 365                                         if (bptr->type == BBTYPE_EXH) {
 366 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 367                                                 if (!IS_INMEMORY(src->flags))
 368                                                         d= src->vv.regoff;
 369                                                 else
 370                                                         d=REG_ITMP1;
 371                                                 M_INTMOVE(REG_ITMP1, d);
 372                                                 emit_store(jd, NULL, src, d);
 373                                         }
 374                                 }
 375                         }
 376
 377                 } else {
 378 #endif
 379
 380                 while (len) {
 381                         len--;
 382                         var = VAR(bptr->invars[len]);
 383                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 384                                 if (bptr->type == BBTYPE_EXH) {
 385                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 386                                         M_INTMOVE(REG_ITMP1, d);
 387                                         emit_store(jd, NULL, var, d);
 388                                 }
 389                         }
 390                         else {
 391                                 assert((var->flags & INOUT));
 392                         }
 393                 }
 394 #if defined(ENABLE_LSRA)
 395                 }
 396 #endif
 397                 /* walk through all instructions */
 398
 399                 len = bptr->icount;
 400                 currentline = 0;
 401
 402                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 403                         if (iptr->line != currentline) {
 404                                 dseg_addlinenumber(cd, iptr->line);
 405                                 currentline = iptr->line;
 406                         }
 407
 408                         MCODECHECK(1024);                         /* 1KB should be enough */
 409
 410                 switch (iptr->opc) {
 411                 case ICMD_NOP:        /* ...  ==> ...                                 */
 412                 case ICMD_POP:        /* ..., value  ==> ...                          */
 413                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 414                         break;
 415
 416                 case ICMD_INLINE_START:
 417
 418                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 419                         break;
 420
 421                 case ICMD_INLINE_BODY:
 422
 423                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 424                         dseg_addlinenumber_inline_start(cd, iptr);
 425                         dseg_addlinenumber(cd, iptr->line);
 426                         break;
 427
 428                 case ICMD_INLINE_END:
 429
 430                         dseg_addlinenumber_inline_end(cd, iptr);
 431                         dseg_addlinenumber(cd, iptr->line);
 432                         break;
 433
 434                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 435
 436                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 437                         emit_nullpointer_check(cd, iptr, s1);
 438                         break;
 439
 440                 /* constant operations ************************************************/
 441
 442                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 443
 444                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 445                         ICONST(d, iptr->sx.val.i);
 446                         emit_store_dst(jd, iptr, d);
 447                         break;
 448
 449                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 450
 451                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 452                         LCONST(d, iptr->sx.val.l);
 453                         emit_store_dst(jd, iptr, d);
 454                         break;
 455
 456                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 457
 458                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 459                         disp = dseg_add_float(cd, iptr->sx.val.f);
 460                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 461                         emit_store_dst(jd, iptr, d);
 462                         break;
 463
 464                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 465
 466                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 467                         disp = dseg_add_double(cd, iptr->sx.val.d);
 468                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 469                         emit_store_dst(jd, iptr, d);
 470                         break;
 471
 472                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 473
 474                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 475
 476                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 477                                 cr   = iptr->sx.val.c.ref;
 478                                 disp = dseg_add_unique_address(cd, cr);
 479
 480 /*                              PROFILE_CYCLE_STOP; */
 481
 482                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
 483                                                                           cr, disp);
 484
 485 /*                              PROFILE_CYCLE_START; */
 486
 487                                 M_ALD(d, RIP, disp);
 488                         }
 489                         else {
 490                                 if (iptr->sx.val.anyptr == 0) {
 491                                         M_CLR(d);
 492                                 }
 493                                 else {
 494                                         disp = dseg_add_address(cd, iptr->sx.val.anyptr);
 495                                         M_ALD(d, RIP, disp);
 496                                 }
 497                         }
 498                         emit_store_dst(jd, iptr, d);
 499                         break;
 500
 501
 502                 /* load/store/copy/move operations ************************************/
 503
 504                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 505                 case ICMD_ALOAD:      /* s1 = local variable                          */
 506                 case ICMD_LLOAD:
 507                 case ICMD_FLOAD:
 508                 case ICMD_DLOAD:
 509                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 510                 case ICMD_LSTORE:
 511                 case ICMD_FSTORE:
 512                 case ICMD_DSTORE:
 513                 case ICMD_COPY:
 514                 case ICMD_MOVE:
 515
 516                         emit_copy(jd, iptr);
 517                         break;
 518
 519                 case ICMD_ASTORE:
 520                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 521                                 emit_copy(jd, iptr);
 522                         break;
 523
 524                 /* integer operations *************************************************/
 525
 526                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 527
 528                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 529                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 530                         M_INTMOVE(s1, d);
 531                         M_INEG(d);
 532                         emit_store_dst(jd, iptr, d);
 533                         break;
 534
 535                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 536
 537                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 538                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 539                         M_INTMOVE(s1, d);
 540                         M_LNEG(d);
 541                         emit_store_dst(jd, iptr, d);
 542                         break;
 543
 544                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 545
 546                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 547                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 548                         M_ISEXT(s1, d);
 549                         emit_store_dst(jd, iptr, d);
 550                         break;
 551
 552                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 553
 554                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 555                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 556                         M_IMOV(s1, d);
 557                         emit_store_dst(jd, iptr, d);
 558                         break;
 559
 560                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 561
 562                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 563                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 564                         M_BSEXT(s1, d);
 565                         emit_store_dst(jd, iptr, d);
 566                         break;
 567
 568                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 569
 570                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 571                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 572                         M_CZEXT(s1, d);
 573                         emit_store_dst(jd, iptr, d);
 574                         break;
 575
 576                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 577
 578                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 579                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 580                         M_SSEXT(s1, d);
 581                         emit_store_dst(jd, iptr, d);
 582                         break;
 583
 584
 585                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 586
 587                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 588                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 589                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 590                         if (s2 == d)
 591                                 M_IADD(s1, d);
 592                         else {
 593                                 M_INTMOVE(s1, d);
 594                                 M_IADD(s2, d);
 595                         }
 596                         emit_store_dst(jd, iptr, d);
 597                         break;
 598
 599                 case ICMD_IINC:
 600                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 601                                       /* sx.val.i = constant                             */
 602
 603                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 604                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 605
 606                         /* Using inc and dec is not faster than add (tested with
 607                            sieve). */
 608
 609                         M_INTMOVE(s1, d);
 610                         M_IADD_IMM(iptr->sx.val.i, d);
 611                         emit_store_dst(jd, iptr, d);
 612                         break;
 613
 614                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 615
 616                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 617                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 618                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 619                         if (s2 == d)
 620                                 M_LADD(s1, d);
 621                         else {
 622                                 M_INTMOVE(s1, d);
 623                                 M_LADD(s2, d);
 624                         }
 625                         emit_store_dst(jd, iptr, d);
 626                         break;
 627
 628                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 629                                       /* sx.val.l = constant                             */
 630
 631                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 632                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 633                         M_INTMOVE(s1, d);
 634                         if (IS_IMM32(iptr->sx.val.l))
 635                                 M_LADD_IMM(iptr->sx.val.l, d);
 636                         else {
 637                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 638                                 M_LADD(REG_ITMP2, d);
 639                         }
 640                         emit_store_dst(jd, iptr, d);
 641                         break;
 642
 643                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 644
 645                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 646                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 647                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 648                         if (s2 == d) {
 649                                 M_INTMOVE(s1, REG_ITMP1);
 650                                 M_ISUB(s2, REG_ITMP1);
 651                                 M_INTMOVE(REG_ITMP1, d);
 652                         } else {
 653                                 M_INTMOVE(s1, d);
 654                                 M_ISUB(s2, d);
 655                         }
 656                         emit_store_dst(jd, iptr, d);
 657                         break;
 658
 659                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 660                                       /* sx.val.i = constant                             */
 661
 662                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 663                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 664                         M_INTMOVE(s1, d);
 665                         M_ISUB_IMM(iptr->sx.val.i, d);
 666                         emit_store_dst(jd, iptr, d);
 667                         break;
 668
 669                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 670
 671                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 672                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 673                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 674                         if (s2 == d) {
 675                                 M_INTMOVE(s1, REG_ITMP1);
 676                                 M_LSUB(s2, REG_ITMP1);
 677                                 M_INTMOVE(REG_ITMP1, d);
 678                         } else {
 679                                 M_INTMOVE(s1, d);
 680                                 M_LSUB(s2, d);
 681                         }
 682                         emit_store_dst(jd, iptr, d);
 683                         break;
 684
 685                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 686                                       /* sx.val.l = constant                             */
 687
 688                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 689                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 690                         M_INTMOVE(s1, d);
 691                         if (IS_IMM32(iptr->sx.val.l))
 692                                 M_LSUB_IMM(iptr->sx.val.l, d);
 693                         else {
 694                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 695                                 M_LSUB(REG_ITMP2, d);
 696                         }
 697                         emit_store_dst(jd, iptr, d);
 698                         break;
 699
 700                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 701
 702                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 703                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 704                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 705                         if (s2 == d)
 706                                 M_IMUL(s1, d);
 707                         else {
 708                                 M_INTMOVE(s1, d);
 709                                 M_IMUL(s2, d);
 710                         }
 711                         emit_store_dst(jd, iptr, d);
 712                         break;
 713
 714                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 715                                       /* sx.val.i = constant                             */
 716
 717                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 718                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 719                         if (iptr->sx.val.i == 2) {
 720                                 M_INTMOVE(s1, d);
 721                                 M_ISLL_IMM(1, d);
 722                         } else
 723                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 724                         emit_store_dst(jd, iptr, d);
 725                         break;
 726
 727                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 728
 729                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 730                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 731                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 732                         if (s2 == d)
 733                                 M_LMUL(s1, d);
 734                         else {
 735                                 M_INTMOVE(s1, d);
 736                                 M_LMUL(s2, d);
 737                         }
 738                         emit_store_dst(jd, iptr, d);
 739                         break;
 740
 741                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 742                                       /* sx.val.l = constant                             */
 743
 744                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 745                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 746                         if (IS_IMM32(iptr->sx.val.l))
 747                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 748                         else {
 749                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 750                                 M_INTMOVE(s1, d);
 751                                 M_LMUL(REG_ITMP2, d);
 752                         }
 753                         emit_store_dst(jd, iptr, d);
 754                         break;
 755
 756                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 757
 758                         s1 = emit_load_s1(jd, iptr, RAX);
 759                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 760                         d = codegen_reg_of_dst(jd, iptr, RAX);
 761
 762                         M_INTMOVE(s1, RAX);
 763                         M_INTMOVE(s2, REG_ITMP3);
 764                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 765
 766                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 767
 768                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 769                         M_BNE(4 + 6);
 770                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 771                         M_BEQ(1 + 3);                                          /* 6 bytes */
 772
 773                         emit_cltd(cd);                                         /* 1 byte  */
 774                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 775
 776                         M_INTMOVE(RAX, d);
 777                         emit_store_dst(jd, iptr, d);
 778                         dst = VAROP(iptr->dst);
 779                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 780                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 781                         break;
 782
 783                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 784
 785                         s1 = emit_load_s1(jd, iptr, RAX);
 786                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 787                         d = codegen_reg_of_dst(jd, iptr, RDX);
 788
 789                         M_INTMOVE(s1, RAX);
 790                         M_INTMOVE(s2, REG_ITMP3);
 791                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 792
 793                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 794
 795                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 796                         M_BNE(3 + 4 + 6);
 797                         M_CLR(RDX);                                            /* 3 bytes */
 798                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 799                         M_BEQ(1 + 3);                                          /* 6 bytes */
 800
 801                         emit_cltd(cd);                                         /* 1 byte  */
 802                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 803
 804                         M_INTMOVE(RDX, d);
 805                         emit_store_dst(jd, iptr, d);
 806                         dst = VAROP(iptr->dst);
 807                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 808                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 809                         break;
 810
 811                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 812                                       /* sx.val.i = constant                             */
 813
 814                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 815                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 816                         M_INTMOVE(s1, REG_ITMP1);
 817                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 818                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 819                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 820                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 821                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 822                         emit_store_dst(jd, iptr, d);
 823                         break;
 824
 825                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 826                                       /* sx.val.i = constant                             */
 827
 828                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 829                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 830                         M_INTMOVE(s1, REG_ITMP1);
 831                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 832                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 833                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 834                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 835                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 836                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 837                         emit_store_dst(jd, iptr, d);
 838                         break;
 839
 840
 841                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 842
 843                         s1 = emit_load_s1(jd, iptr, RAX);
 844                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 845                         d = codegen_reg_of_dst(jd, iptr, RAX);
 846
 847                         M_INTMOVE(s1, RAX);
 848                         M_INTMOVE(s2, REG_ITMP3);
 849                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 850
 851                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 852
 853                         /* check as described in jvm spec */
 854                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 855                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 856                         M_BNE(4 + 6);
 857                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 858                         M_BEQ(2 + 3);                                          /* 6 bytes */
 859
 860                         emit_cqto(cd);                                         /* 2 bytes */
 861                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 862
 863                         M_INTMOVE(RAX, d);
 864                         emit_store_dst(jd, iptr, d);
 865                         dst = VAROP(iptr->dst);
 866                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 867                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 868                         break;
 869
 870                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 871
 872                         s1 = emit_load_s1(jd, iptr, RAX);
 873                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 874                         d = codegen_reg_of_dst(jd, iptr, RDX);
 875
 876                         M_INTMOVE(s1, RAX);
 877                         M_INTMOVE(s2, REG_ITMP3);
 878                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 879
 880                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 881
 882                         /* check as described in jvm spec */
 883                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 884                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 885                         M_BNE(3 + 4 + 6);
 886                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 887                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 888                         M_BEQ(2 + 3);                                          /* 6 bytes */
 889
 890                         emit_cqto(cd);                                         /* 2 bytes */
 891                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 892
 893                         M_INTMOVE(RDX, d);
 894                         emit_store_dst(jd, iptr, d);
 895                         dst = VAROP(iptr->dst);
 896                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 897                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 898                         break;
 899
 900                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 901                                       /* sx.val.i = constant                             */
 902
 903                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 904                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 905                         M_INTMOVE(s1, REG_ITMP1);
 906                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 907                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 908                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 909                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 910                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 911                         emit_store_dst(jd, iptr, d);
 912                         break;
 913
 914                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 915                                       /* sx.val.l = constant                             */
 916
 917                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 918                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 919                         M_INTMOVE(s1, REG_ITMP1);
 920                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 921                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 922                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 923                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 924                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 925                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 926                         emit_store_dst(jd, iptr, d);
 927                         break;
 928
 929                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 930
 931                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 932                         emit_ishift(jd, SHIFT_SHL, iptr);
 933                         break;
 934
 935                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 936                                       /* sx.val.i = constant                             */
 937
 938                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 939                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 940                         M_INTMOVE(s1, d);
 941                         M_ISLL_IMM(iptr->sx.val.i, d);
 942                         emit_store_dst(jd, iptr, d);
 943                         break;
 944
 945                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 946
 947                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 948                         emit_ishift(jd, SHIFT_SAR, iptr);
 949                         break;
 950
 951                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 952                                       /* sx.val.i = constant                             */
 953
 954                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 955                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 956                         M_INTMOVE(s1, d);
 957                         M_ISRA_IMM(iptr->sx.val.i, d);
 958                         emit_store_dst(jd, iptr, d);
 959                         break;
 960
 961                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 962
 963                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 964                         emit_ishift(jd, SHIFT_SHR, iptr);
 965                         break;
 966
 967                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 968                                       /* sx.val.i = constant                             */
 969
 970                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 971                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 972                         M_INTMOVE(s1, d);
 973                         M_ISRL_IMM(iptr->sx.val.i, d);
 974                         emit_store_dst(jd, iptr, d);
 975                         break;
 976
 977                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 978
 979                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 980                         emit_lshift(jd, SHIFT_SHL, iptr);
 981                         break;
 982
 983         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 984                                           /* sx.val.i = constant                             */
 985
 986                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 987                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 988                         M_INTMOVE(s1, d);
 989                         M_LSLL_IMM(iptr->sx.val.i, d);
 990                         emit_store_dst(jd, iptr, d);
 991                         break;
 992
 993                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 994
 995                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 996                         emit_lshift(jd, SHIFT_SAR, iptr);
 997                         break;
 998
 999                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1000                                       /* sx.val.i = constant                             */
1001
1002                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1003                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1004                         M_INTMOVE(s1, d);
1005                         M_LSRA_IMM(iptr->sx.val.i, d);
1006                         emit_store_dst(jd, iptr, d);
1007                         break;
1008
1009                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1010
1011                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1012                         emit_lshift(jd, SHIFT_SHR, iptr);
1013                         break;
1014
1015                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1016                                       /* sx.val.l = constant                             */
1017
1018                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1019                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1020                         M_INTMOVE(s1, d);
1021                         M_LSRL_IMM(iptr->sx.val.i, d);
1022                         emit_store_dst(jd, iptr, d);
1023                         break;
1024
1025                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1026
1027                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1028                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1029                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1030                         if (s2 == d)
1031                                 M_IAND(s1, d);
1032                         else {
1033                                 M_INTMOVE(s1, d);
1034                                 M_IAND(s2, d);
1035                         }
1036                         emit_store_dst(jd, iptr, d);
1037                         break;
1038
1039                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1040                                       /* sx.val.i = constant                             */
1041
1042                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1043                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1044                         M_INTMOVE(s1, d);
1045                         M_IAND_IMM(iptr->sx.val.i, d);
1046                         emit_store_dst(jd, iptr, d);
1047                         break;
1048
1049                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1050
1051                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1052                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1053                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1054                         if (s2 == d)
1055                                 M_LAND(s1, d);
1056                         else {
1057                                 M_INTMOVE(s1, d);
1058                                 M_LAND(s2, d);
1059                         }
1060                         emit_store_dst(jd, iptr, d);
1061                         break;
1062
1063                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1064                                       /* sx.val.l = constant                             */
1065
1066                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1067                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1068                         M_INTMOVE(s1, d);
1069                         if (IS_IMM32(iptr->sx.val.l))
1070                                 M_LAND_IMM(iptr->sx.val.l, d);
1071                         else {
1072                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1073                                 M_LAND(REG_ITMP2, d);
1074                         }
1075                         emit_store_dst(jd, iptr, d);
1076                         break;
1077
1078                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1079
1080                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1081                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1082                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1083                         if (s2 == d)
1084                                 M_IOR(s1, d);
1085                         else {
1086                                 M_INTMOVE(s1, d);
1087                                 M_IOR(s2, d);
1088                         }
1089                         emit_store_dst(jd, iptr, d);
1090                         break;
1091
1092                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1093                                       /* sx.val.i = constant                             */
1094
1095                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1096                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1097                         M_INTMOVE(s1, d);
1098                         M_IOR_IMM(iptr->sx.val.i, d);
1099                         emit_store_dst(jd, iptr, d);
1100                         break;
1101
1102                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1103
1104                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1105                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1106                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1107                         if (s2 == d)
1108                                 M_LOR(s1, d);
1109                         else {
1110                                 M_INTMOVE(s1, d);
1111                                 M_LOR(s2, d);
1112                         }
1113                         emit_store_dst(jd, iptr, d);
1114                         break;
1115
1116                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1117                                       /* sx.val.l = constant                             */
1118
1119                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1120                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1121                         M_INTMOVE(s1, d);
1122                         if (IS_IMM32(iptr->sx.val.l))
1123                                 M_LOR_IMM(iptr->sx.val.l, d);
1124                         else {
1125                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1126                                 M_LOR(REG_ITMP2, d);
1127                         }
1128                         emit_store_dst(jd, iptr, d);
1129                         break;
1130
1131                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1132
1133                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1134                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1135                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1136                         if (s2 == d)
1137                                 M_IXOR(s1, d);
1138                         else {
1139                                 M_INTMOVE(s1, d);
1140                                 M_IXOR(s2, d);
1141                         }
1142                         emit_store_dst(jd, iptr, d);
1143                         break;
1144
1145                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1146                                       /* sx.val.i = constant                             */
1147
1148                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1149                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1150                         M_INTMOVE(s1, d);
1151                         M_IXOR_IMM(iptr->sx.val.i, d);
1152                         emit_store_dst(jd, iptr, d);
1153                         break;
1154
1155                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1156
1157                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1158                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1159                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1160                         if (s2 == d)
1161                                 M_LXOR(s1, d);
1162                         else {
1163                                 M_INTMOVE(s1, d);
1164                                 M_LXOR(s2, d);
1165                         }
1166                         emit_store_dst(jd, iptr, d);
1167                         break;
1168
1169                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1170                                       /* sx.val.l = constant                             */
1171
1172                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1173                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1174                         M_INTMOVE(s1, d);
1175                         if (IS_IMM32(iptr->sx.val.l))
1176                                 M_LXOR_IMM(iptr->sx.val.l, d);
1177                         else {
1178                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1179                                 M_LXOR(REG_ITMP2, d);
1180                         }
1181                         emit_store_dst(jd, iptr, d);
1182                         break;
1183
1184
1185                 /* floating operations ************************************************/
1186
1187                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1188
1189                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1190                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1191                         disp = dseg_add_s4(cd, 0x80000000);
1192                         M_FLTMOVE(s1, d);
1193                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1194                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1195                         emit_store_dst(jd, iptr, d);
1196                         break;
1197
1198                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1199
1200                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1201                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1202                         disp = dseg_add_s8(cd, 0x8000000000000000);
1203                         M_FLTMOVE(s1, d);
1204                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1205                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1206                         emit_store_dst(jd, iptr, d);
1207                         break;
1208
1209                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1210
1211                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1212                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1213                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1214                         if (s2 == d)
1215                                 M_FADD(s1, d);
1216                         else {
1217                                 M_FLTMOVE(s1, d);
1218                                 M_FADD(s2, d);
1219                         }
1220                         emit_store_dst(jd, iptr, d);
1221                         break;
1222
1223                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1224
1225                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1226                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1227                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1228                         if (s2 == d)
1229                                 M_DADD(s1, d);
1230                         else {
1231                                 M_FLTMOVE(s1, d);
1232                                 M_DADD(s2, d);
1233                         }
1234                         emit_store_dst(jd, iptr, d);
1235                         break;
1236
1237                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1238
1239                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1240                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1241                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1242                         if (s2 == d) {
1243                                 M_FLTMOVE(s2, REG_FTMP2);
1244                                 s2 = REG_FTMP2;
1245                         }
1246                         M_FLTMOVE(s1, d);
1247                         M_FSUB(s2, d);
1248                         emit_store_dst(jd, iptr, d);
1249                         break;
1250
1251                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1252
1253                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1254                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1255                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1256                         if (s2 == d) {
1257                                 M_FLTMOVE(s2, REG_FTMP2);
1258                                 s2 = REG_FTMP2;
1259                         }
1260                         M_FLTMOVE(s1, d);
1261                         M_DSUB(s2, d);
1262                         emit_store_dst(jd, iptr, d);
1263                         break;
1264
1265                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1266
1267                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1268                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1269                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1270                         if (s2 == d)
1271                                 M_FMUL(s1, d);
1272                         else {
1273                                 M_FLTMOVE(s1, d);
1274                                 M_FMUL(s2, d);
1275                         }
1276                         emit_store_dst(jd, iptr, d);
1277                         break;
1278
1279                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1280
1281                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1282                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1283                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1284                         if (s2 == d)
1285                                 M_DMUL(s1, d);
1286                         else {
1287                                 M_FLTMOVE(s1, d);
1288                                 M_DMUL(s2, d);
1289                         }
1290                         emit_store_dst(jd, iptr, d);
1291                         break;
1292
1293                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1294
1295                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1296                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1297                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1298                         if (s2 == d) {
1299                                 M_FLTMOVE(s2, REG_FTMP2);
1300                                 s2 = REG_FTMP2;
1301                         }
1302                         M_FLTMOVE(s1, d);
1303                         M_FDIV(s2, d);
1304                         emit_store_dst(jd, iptr, d);
1305                         break;
1306
1307                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1308
1309                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1310                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1311                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1312                         if (s2 == d) {
1313                                 M_FLTMOVE(s2, REG_FTMP2);
1314                                 s2 = REG_FTMP2;
1315                         }
1316                         M_FLTMOVE(s1, d);
1317                         M_DDIV(s2, d);
1318                         emit_store_dst(jd, iptr, d);
1319                         break;
1320
1321                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1322
1323                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1324                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1325                         M_CVTIF(s1, d);
1326                         emit_store_dst(jd, iptr, d);
1327                         break;
1328
1329                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1330
1331                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1332                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1333                         M_CVTID(s1, d);
1334                         emit_store_dst(jd, iptr, d);
1335                         break;
1336
1337                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1338
1339                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1340                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1341                         M_CVTLF(s1, d);
1342                         emit_store_dst(jd, iptr, d);
1343                         break;
1344
1345                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1346
1347                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1348                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1349                         M_CVTLD(s1, d);
1350                         emit_store_dst(jd, iptr, d);
1351                         break;
1352
1353                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1354
1355                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1356                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1357                         M_CVTFI(s1, d);
1358                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1359                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1360                                 ((REG_RESULT == d) ? 0 : 3);
1361                         M_BNE(disp);
1362                         M_FLTMOVE(s1, REG_FTMP1);
1363                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1364                         M_CALL(REG_ITMP2);
1365                         M_INTMOVE(REG_RESULT, d);
1366                         emit_store_dst(jd, iptr, d);
1367                         break;
1368
1369                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1370
1371                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1372                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1373                         M_CVTDI(s1, d);
1374                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1375                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1376                                 ((REG_RESULT == d) ? 0 : 3);
1377                         M_BNE(disp);
1378                         M_FLTMOVE(s1, REG_FTMP1);
1379                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1380                         M_CALL(REG_ITMP2);
1381                         M_INTMOVE(REG_RESULT, d);
1382                         emit_store_dst(jd, iptr, d);
1383                         break;
1384
1385                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1386
1387                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1388                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1389                         M_CVTFL(s1, d);
1390                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1391                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1392                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1393                                 ((REG_RESULT == d) ? 0 : 3);
1394                         M_BNE(disp);
1395                         M_FLTMOVE(s1, REG_FTMP1);
1396                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1397                         M_CALL(REG_ITMP2);
1398                         M_INTMOVE(REG_RESULT, d);
1399                         emit_store_dst(jd, iptr, d);
1400                         break;
1401
1402                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1403
1404                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1405                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1406                         M_CVTDL(s1, d);
1407                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1408                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1409                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1410                                 ((REG_RESULT == d) ? 0 : 3);
1411                         M_BNE(disp);
1412                         M_FLTMOVE(s1, REG_FTMP1);
1413                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1414                         M_CALL(REG_ITMP2);
1415                         M_INTMOVE(REG_RESULT, d);
1416                         emit_store_dst(jd, iptr, d);
1417                         break;
1418
1419                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1420
1421                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1422                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1423                         M_CVTFD(s1, d);
1424                         emit_store_dst(jd, iptr, d);
1425                         break;
1426
1427                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1428
1429                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1430                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1431                         M_CVTDF(s1, d);
1432                         emit_store_dst(jd, iptr, d);
1433                         break;
1434
1435                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1436                                           /* == => 0, < => 1, > => -1 */
1437
1438                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1439                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1440                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1441                         M_CLR(d);
1442                         M_MOV_IMM(1, REG_ITMP1);
1443                         M_MOV_IMM(-1, REG_ITMP2);
1444                         emit_ucomiss_reg_reg(cd, s1, s2);
1445                         M_CMOVULT(REG_ITMP1, d);
1446                         M_CMOVUGT(REG_ITMP2, d);
1447                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1448                         emit_store_dst(jd, iptr, d);
1449                         break;
1450
1451                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1452                                           /* == => 0, < => 1, > => -1 */
1453
1454                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1455                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1456                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1457                         M_CLR(d);
1458                         M_MOV_IMM(1, REG_ITMP1);
1459                         M_MOV_IMM(-1, REG_ITMP2);
1460                         emit_ucomiss_reg_reg(cd, s1, s2);
1461                         M_CMOVULT(REG_ITMP1, d);
1462                         M_CMOVUGT(REG_ITMP2, d);
1463                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1464                         emit_store_dst(jd, iptr, d);
1465                         break;
1466
1467                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1468                                           /* == => 0, < => 1, > => -1 */
1469
1470                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1471                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1472                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1473                         M_CLR(d);
1474                         M_MOV_IMM(1, REG_ITMP1);
1475                         M_MOV_IMM(-1, REG_ITMP2);
1476                         emit_ucomisd_reg_reg(cd, s1, s2);
1477                         M_CMOVULT(REG_ITMP1, d);
1478                         M_CMOVUGT(REG_ITMP2, d);
1479                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1480                         emit_store_dst(jd, iptr, d);
1481                         break;
1482
1483                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1484                                           /* == => 0, < => 1, > => -1 */
1485
1486                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1487                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1488                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1489                         M_CLR(d);
1490                         M_MOV_IMM(1, REG_ITMP1);
1491                         M_MOV_IMM(-1, REG_ITMP2);
1492                         emit_ucomisd_reg_reg(cd, s1, s2);
1493                         M_CMOVULT(REG_ITMP1, d);
1494                         M_CMOVUGT(REG_ITMP2, d);
1495                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1496                         emit_store_dst(jd, iptr, d);
1497                         break;
1498
1499
1500                 /* memory operations **************************************************/
1501
1502                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1503
1504                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1505                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1506                         /* implicit null-pointer check */
1507                         M_ILD(d, s1, OFFSET(java_array_t, size));
1508                         emit_store_dst(jd, iptr, d);
1509                         break;
1510
1511                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1512
1513                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1514                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1515                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1516                         /* implicit null-pointer check */
1517                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1518                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1519                         emit_store_dst(jd, iptr, d);
1520                         break;
1521
1522                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1523
1524                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1525                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1526                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1527                         /* implicit null-pointer check */
1528                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1529                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1530                         emit_store_dst(jd, iptr, d);
1531                         break;
1532
1533                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1534
1535                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1536                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1537                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1538                         /* implicit null-pointer check */
1539                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1540                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1541                         emit_store_dst(jd, iptr, d);
1542                         break;
1543
1544                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1545
1546                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1547                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1548                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1549                         /* implicit null-pointer check */
1550                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1551                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1552                         emit_store_dst(jd, iptr, d);
1553                         break;
1554
1555                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1556
1557                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1558                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1559                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1560                         /* implicit null-pointer check */
1561                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1562                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1563                         emit_store_dst(jd, iptr, d);
1564                         break;
1565
1566                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1567
1568                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1569                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1570                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1571                         /* implicit null-pointer check */
1572                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1573                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1574                         emit_store_dst(jd, iptr, d);
1575                         break;
1576
1577                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1578
1579                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1580                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1581                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1582                         /* implicit null-pointer check */
1583                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1584                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1585                         emit_store_dst(jd, iptr, d);
1586                         break;
1587
1588                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1589
1590                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1591                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1592                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1593                         /* implicit null-pointer check */
1594                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1595                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1596                         emit_store_dst(jd, iptr, d);
1597                         break;
1598
1599
1600                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1601
1602                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1603                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1604                         /* implicit null-pointer check */
1605                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1606                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1607                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1608                         break;
1609
1610                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1611
1612                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1613                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1614                         /* implicit null-pointer check */
1615                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1616                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1617                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1618                         break;
1619
1620                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1621
1622                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1623                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1624                         /* implicit null-pointer check */
1625                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1626                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1627                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1628                         break;
1629
1630                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1631
1632                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1633                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1634                         /* implicit null-pointer check */
1635                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1636                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1637                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1638                         break;
1639
1640                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1641
1642                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1643                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1644                         /* implicit null-pointer check */
1645                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1646                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1647                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1648                         break;
1649
1650                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1651
1652                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1653                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1654                         /* implicit null-pointer check */
1655                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1656                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1657                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1658                         break;
1659
1660                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1661
1662                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1663                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1664                         /* implicit null-pointer check */
1665                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1666                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1667                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1668                         break;
1669
1670                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1671
1672                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1673                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1674                         /* implicit null-pointer check */
1675                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1676                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1677
1678                         M_MOV(s1, REG_A0);
1679                         M_MOV(s3, REG_A1);
1680                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
1681                         M_CALL(REG_ITMP1);
1682                         emit_arraystore_check(cd, iptr);
1683
1684                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1685                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1686                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1687                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1688                         break;
1689
1690
1691                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1692
1693                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1694                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1695                         /* implicit null-pointer check */
1696                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1697                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1698                         break;
1699
1700                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1701
1702                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1703                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1704                         /* implicit null-pointer check */
1705                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1706                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1707                         break;
1708
1709                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1710
1711                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1712                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1713                         /* implicit null-pointer check */
1714                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1715                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1716                         break;
1717
1718                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1719
1720                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1721                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1722                         /* implicit null-pointer check */
1723                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1724                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1725                         break;
1726
1727                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1728
1729                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1730                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1731                         /* implicit null-pointer check */
1732                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1733
1734                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1735                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1736                         }
1737                         else {
1738                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1739                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1740                         }
1741                         break;
1742
1743                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1744
1745                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1746                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1747                         /* implicit null-pointer check */
1748                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1749                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1750                         break;
1751
1752
1753                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1754
1755                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1756                                 uf        = iptr->sx.s23.s3.uf;
1757                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1758                                 disp      = dseg_add_unique_address(cd, uf);
1759
1760 /*                              PROFILE_CYCLE_STOP; */
1761
1762                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1763
1764 /*                              PROFILE_CYCLE_START; */
1765                         }
1766                         else {
1767                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1768                                 fieldtype = fi->type;
1769                                 disp      = dseg_add_address(cd, fi->value);
1770
1771                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1772                                         PROFILE_CYCLE_STOP;
1773
1774                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1775                                                                                   fi->class, 0);
1776
1777                                         PROFILE_CYCLE_START;
1778                                 }
1779                         }
1780
1781                         /* This approach is much faster than moving the field
1782                            address inline into a register. */
1783
1784                         M_ALD(REG_ITMP1, RIP, disp);
1785
1786                         switch (fieldtype) {
1787                         case TYPE_INT:
1788                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1789                                 M_ILD(d, REG_ITMP1, 0);
1790                                 break;
1791                         case TYPE_LNG:
1792                         case TYPE_ADR:
1793                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1794                                 M_LLD(d, REG_ITMP1, 0);
1795                                 break;
1796                         case TYPE_FLT:
1797                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1798                                 M_FLD(d, REG_ITMP1, 0);
1799                                 break;
1800                         case TYPE_DBL:
1801                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1802                                 M_DLD(d, REG_ITMP1, 0);
1803                                 break;
1804                         }
1805                         emit_store_dst(jd, iptr, d);
1806                         break;
1807
1808                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1809
1810                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1811                                 uf        = iptr->sx.s23.s3.uf;
1812                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1813                                 disp      = dseg_add_unique_address(cd, uf);
1814
1815 /*                              PROFILE_CYCLE_STOP; */
1816
1817                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1818
1819 /*                              PROFILE_CYCLE_START; */
1820                         }
1821                         else {
1822                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1823                                 fieldtype = fi->type;
1824                                 disp      = dseg_add_address(cd, fi->value);
1825
1826                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1827                                         PROFILE_CYCLE_STOP;
1828
1829                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1830                                                                                   fi->class, 0);
1831
1832                                         PROFILE_CYCLE_START;
1833                                 }
1834                         }
1835
1836                         /* This approach is much faster than moving the field
1837                            address inline into a register. */
1838
1839                         M_ALD(REG_ITMP1, RIP, disp);
1840
1841                         switch (fieldtype) {
1842                         case TYPE_INT:
1843                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1844                                 M_IST(s1, REG_ITMP1, 0);
1845                                 break;
1846                         case TYPE_LNG:
1847                         case TYPE_ADR:
1848                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1849                                 M_LST(s1, REG_ITMP1, 0);
1850                                 break;
1851                         case TYPE_FLT:
1852                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1853                                 M_FST(s1, REG_ITMP1, 0);
1854                                 break;
1855                         case TYPE_DBL:
1856                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1857                                 M_DST(s1, REG_ITMP1, 0);
1858                                 break;
1859                         }
1860                         break;
1861
1862                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1863                                           /* val = value (in current instruction)     */
1864                                           /* following NOP)                           */
1865
1866                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1867                                 uf        = iptr->sx.s23.s3.uf;
1868                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1869                                 disp      = dseg_add_unique_address(cd, uf);
1870
1871 /*                              PROFILE_CYCLE_STOP; */
1872
1873                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1874
1875 /*                              PROFILE_CYCLE_START; */
1876                         }
1877                         else {
1878                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1879                                 fieldtype = fi->type;
1880                                 disp      = dseg_add_address(cd, fi->value);
1881
1882                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1883                                         PROFILE_CYCLE_STOP;
1884
1885                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1886                                                                                   fi->class, 0);
1887
1888                                         PROFILE_CYCLE_START;
1889                                 }
1890                         }
1891
1892                         /* This approach is much faster than moving the field
1893                            address inline into a register. */
1894
1895                         M_ALD(REG_ITMP1, RIP, disp);
1896
1897                         switch (fieldtype) {
1898                         case TYPE_INT:
1899                         case TYPE_FLT:
1900                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1901                                 break;
1902                         case TYPE_LNG:
1903                         case TYPE_ADR:
1904                         case TYPE_DBL:
1905                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1906                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1907                                 else {
1908                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1909                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1910                                 }
1911                                 break;
1912                         }
1913                         break;
1914
1915                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1916
1917                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1918
1919                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1920                                 uf        = iptr->sx.s23.s3.uf;
1921                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1922                                 disp      = 0;
1923
1924 /*                              PROFILE_CYCLE_STOP; */
1925
1926                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1927
1928 /*                              PROFILE_CYCLE_START; */
1929                         }
1930                         else {
1931                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1932                                 fieldtype = fi->type;
1933                                 disp      = fi->offset;
1934                         }
1935
1936                         /* implicit null-pointer check */
1937                         switch (fieldtype) {
1938                         case TYPE_INT:
1939                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1940                                 M_ILD32(d, s1, disp);
1941                                 break;
1942                         case TYPE_LNG:
1943                         case TYPE_ADR:
1944                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1945                                 M_LLD32(d, s1, disp);
1946                                 break;
1947                         case TYPE_FLT:
1948                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1949                                 M_FLD32(d, s1, disp);
1950                                 break;
1951                         case TYPE_DBL:
1952                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1953                                 M_DLD32(d, s1, disp);
1954                                 break;
1955                         }
1956                         emit_store_dst(jd, iptr, d);
1957                         break;
1958
1959                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1960
1961                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1962                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1963
1964                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1965                                 uf        = iptr->sx.s23.s3.uf;
1966                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1967                                 disp      = 0;
1968
1969 /*                              PROFILE_CYCLE_STOP; */
1970
1971                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1972
1973 /*                              PROFILE_CYCLE_START; */
1974                         }
1975                         else {
1976                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1977                                 fieldtype = fi->type;
1978                                 disp      = fi->offset;
1979                         }
1980
1981                         /* implicit null-pointer check */
1982                         switch (fieldtype) {
1983                         case TYPE_INT:
1984                                 M_IST32(s2, s1, disp);
1985                                 break;
1986                         case TYPE_LNG:
1987                         case TYPE_ADR:
1988                                 M_LST32(s2, s1, disp);
1989                                 break;
1990                         case TYPE_FLT:
1991                                 M_FST32(s2, s1, disp);
1992                                 break;
1993                         case TYPE_DBL:
1994                                 M_DST32(s2, s1, disp);
1995                                 break;
1996                         }
1997                         break;
1998
1999                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2000                                           /* val = value (in current instruction)     */
2001                                           /* following NOP)                           */
2002
2003                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2004
2005                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2006                                 uf        = iptr->sx.s23.s3.uf;
2007                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2008                                 disp      = 0;
2009
2010 /*                              PROFILE_CYCLE_STOP; */
2011
2012                                 patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
2013
2014 /*                              PROFILE_CYCLE_START; */
2015                         }
2016                         else {
2017                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2018                                 fieldtype = fi->type;
2019                                 disp      = fi->offset;
2020                         }
2021
2022                         /* implicit null-pointer check */
2023                         switch (fieldtype) {
2024                         case TYPE_INT:
2025                         case TYPE_FLT:
2026                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2027                                 break;
2028                         case TYPE_LNG:
2029                         case TYPE_ADR:
2030                         case TYPE_DBL:
2031                                 /* XXX why no check for IS_IMM32? */
2032                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2033                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2034                                 break;
2035                         }
2036                         break;
2037
2038
2039                 /* branch operations **************************************************/
2040
2041                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2042
2043                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2044                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2045
2046                         PROFILE_CYCLE_STOP;
2047
2048 #ifdef ENABLE_VERIFIER
2049                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2050                                 uc = iptr->sx.s23.s2.uc;
2051
2052                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2053                         }
2054 #endif /* ENABLE_VERIFIER */
2055
2056                         M_CALL_IMM(0);                            /* passing exception pc */
2057                         M_POP(REG_ITMP2_XPC);
2058
2059                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2060                         M_JMP(REG_ITMP3);
2061                         break;
2062
2063                 case ICMD_GOTO:         /* ... ==> ...                                */
2064                 case ICMD_RET:
2065
2066                         emit_br(cd, iptr->dst.block);
2067                         ALIGNCODENOP;
2068                         break;
2069
2070                 case ICMD_JSR:          /* ... ==> ...                                */
2071
2072                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2073                         ALIGNCODENOP;
2074                         break;
2075
2076                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2077                 case ICMD_IFNONNULL:
2078
2079                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2080                         M_TEST(s1);
2081                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2082                         break;
2083
2084                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2085                 case ICMD_IFLT:
2086                 case ICMD_IFLE:
2087                 case ICMD_IFNE:
2088                 case ICMD_IFGT:
2089                 case ICMD_IFGE:
2090
2091                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2092                         M_ICMP_IMM(iptr->sx.val.i, s1);
2093                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2094                         break;
2095
2096                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2097                 case ICMD_IF_LNE:
2098                 case ICMD_IF_LLT:
2099                 case ICMD_IF_LGE:
2100                 case ICMD_IF_LGT:
2101                 case ICMD_IF_LLE:
2102
2103                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2104                         if (IS_IMM32(iptr->sx.val.l))
2105                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2106                         else {
2107                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2108                                 M_LCMP(REG_ITMP2, s1);
2109                         }
2110                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2111                         break;
2112
2113                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2114                 case ICMD_IF_ICMPNE:
2115                 case ICMD_IF_ICMPLT:
2116                 case ICMD_IF_ICMPGE:
2117                 case ICMD_IF_ICMPGT:
2118                 case ICMD_IF_ICMPLE:
2119
2120                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2121                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2122                         M_ICMP(s2, s1);
2123                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2124                         break;
2125
2126                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2127                 case ICMD_IF_ACMPNE:
2128
2129                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2130                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2131                         M_LCMP(s2, s1);
2132                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2133                         break;
2134
2135                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2136                 case ICMD_IF_LCMPNE:
2137                 case ICMD_IF_LCMPLT:
2138                 case ICMD_IF_LCMPGE:
2139                 case ICMD_IF_LCMPGT:
2140                 case ICMD_IF_LCMPLE:
2141
2142                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2143                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2144                         M_LCMP(s2, s1);
2145                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2146                         break;
2147
2148                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2149                 case ICMD_LRETURN:
2150
2151                         REPLACEMENT_POINT_RETURN(cd, iptr);
2152                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2153                         M_INTMOVE(s1, REG_RESULT);
2154                         goto nowperformreturn;
2155
2156                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2157
2158                         REPLACEMENT_POINT_RETURN(cd, iptr);
2159                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2160                         M_INTMOVE(s1, REG_RESULT);
2161
2162 #ifdef ENABLE_VERIFIER
2163                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2164                                 uc = iptr->sx.s23.s2.uc;
2165
2166                                 PROFILE_CYCLE_STOP;
2167
2168                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2169
2170                                 PROFILE_CYCLE_START;
2171                         }
2172 #endif /* ENABLE_VERIFIER */
2173                         goto nowperformreturn;
2174
2175                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2176                 case ICMD_DRETURN:
2177
2178                         REPLACEMENT_POINT_RETURN(cd, iptr);
2179                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2180                         M_FLTMOVE(s1, REG_FRESULT);
2181                         goto nowperformreturn;
2182
2183                 case ICMD_RETURN:      /* ...  ==> ...                                */
2184
2185                         REPLACEMENT_POINT_RETURN(cd, iptr);
2186
2187 nowperformreturn:
2188                         {
2189                         s4 i, p;
2190
2191                         p = cd->stackframesize;
2192
2193 #if !defined(NDEBUG)
2194                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2195                                 emit_verbosecall_exit(jd);
2196 #endif /* !defined(NDEBUG) */
2197
2198 #if defined(ENABLE_THREADS)
2199                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2200                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2201
2202                                 /* we need to save the proper return value */
2203                                 switch (iptr->opc) {
2204                                 case ICMD_IRETURN:
2205                                 case ICMD_ARETURN:
2206                                 case ICMD_LRETURN:
2207                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2208                                         break;
2209                                 case ICMD_FRETURN:
2210                                 case ICMD_DRETURN:
2211                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2212                                         break;
2213                                 }
2214
2215                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2216                                 M_CALL(REG_ITMP1);
2217
2218                                 /* and now restore the proper return value */
2219                                 switch (iptr->opc) {
2220                                 case ICMD_IRETURN:
2221                                 case ICMD_ARETURN:
2222                                 case ICMD_LRETURN:
2223                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2224                                         break;
2225                                 case ICMD_FRETURN:
2226                                 case ICMD_DRETURN:
2227                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2228                                         break;
2229                                 }
2230                         }
2231 #endif
2232
2233                         /* restore saved registers */
2234
2235                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2236                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2237                         }
2238                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2239                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2240                         }
2241
2242                         /* deallocate stack */
2243
2244                         if (cd->stackframesize)
2245                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2246
2247                         /* generate method profiling code */
2248
2249                         PROFILE_CYCLE_STOP;
2250
2251                         M_RET;
2252                         }
2253                         break;
2254
2255
2256                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2257                         {
2258                                 s4 i, l;
2259                                 branch_target_t *table;
2260
2261                                 table = iptr->dst.table;
2262
2263                                 l = iptr->sx.s23.s2.tablelow;
2264                                 i = iptr->sx.s23.s3.tablehigh;
2265
2266                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2267                                 M_INTMOVE(s1, REG_ITMP1);
2268
2269                                 if (l != 0)
2270                                         M_ISUB_IMM(l, REG_ITMP1);
2271
2272                                 /* number of targets */
2273                                 i = i - l + 1;
2274
2275                 /* range check */
2276
2277                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2278                                 emit_bugt(cd, table[0].block);
2279
2280                                 /* build jump table top down and use address of lowest entry */
2281
2282                                 table += i;
2283
2284                                 while (--i >= 0) {
2285                                         dseg_add_target(cd, table->block);
2286                                         --table;
2287                                 }
2288
2289                                 /* length of dataseg after last dseg_add_target is used
2290                                    by load */
2291
2292                                 M_MOV_IMM(0, REG_ITMP2);
2293                                 dseg_adddata(cd);
2294                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2295                                 M_JMP(REG_ITMP1);
2296                         }
2297                         break;
2298
2299
2300                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2301                         {
2302                                 s4 i;
2303                                 lookup_target_t *lookup;
2304
2305                                 lookup = iptr->dst.lookup;
2306
2307                                 i = iptr->sx.s23.s2.lookupcount;
2308
2309                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2310                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2311
2312                                 while (--i >= 0) {
2313                                         M_ICMP_IMM(lookup->value, s1);
2314                                         emit_beq(cd, lookup->target.block);
2315                                         lookup++;
2316                                 }
2317
2318                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2319                                 ALIGNCODENOP;
2320                         }
2321                         break;
2322
2323
2324                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2325
2326                         REPLACEMENT_POINT_FORGC_BUILTIN(cd, iptr);
2327
2328                         bte = iptr->sx.s23.s3.bte;
2329                         md  = bte->md;
2330                         goto gen_method;
2331
2332                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2333
2334                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2335                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2336                 case ICMD_INVOKEINTERFACE:
2337
2338                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2339
2340                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2341                                 lm = NULL;
2342                                 um = iptr->sx.s23.s3.um;
2343                                 md = um->methodref->parseddesc.md;
2344                         }
2345                         else {
2346                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2347                                 um = NULL;
2348                                 md = lm->parseddesc;
2349                         }
2350
2351 gen_method:
2352                         s3 = md->paramcount;
2353
2354                         MCODECHECK((20 * s3) + 128);
2355
2356                         /* copy arguments to registers or stack location */
2357
2358                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2359                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2360                                 d   = md->params[s3].regoff;
2361
2362                                 /* already preallocated (ARGVAR)? */
2363
2364                                 if (var->flags & PREALLOC)
2365                                         continue;
2366
2367                                 if (IS_INT_LNG_TYPE(var->type)) {
2368                                         if (!md->params[s3].inmemory) {
2369                                                 s1 = emit_load(jd, iptr, var, d);
2370                                                 M_INTMOVE(s1, d);
2371                                         }
2372                                         else {
2373                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2374                                                 M_LST(s1, REG_SP, d);
2375                                         }
2376                                 }
2377                                 else {
2378                                         if (!md->params[s3].inmemory) {
2379                                                 s1 = emit_load(jd, iptr, var, d);
2380                                                 M_FLTMOVE(s1, d);
2381                                         }
2382                                         else {
2383                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2384
2385                                                 if (IS_2_WORD_TYPE(var->type))
2386                                                         M_DST(s1, REG_SP, d);
2387                                                 else
2388                                                         M_FST(s1, REG_SP, d);
2389                                         }
2390                                 }
2391                         }
2392
2393                         /* generate method profiling code */
2394
2395                         PROFILE_CYCLE_STOP;
2396
2397                         switch (iptr->opc) {
2398                         case ICMD_BUILTIN:
2399                                 if (bte->stub == NULL) {
2400                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2401                                 }
2402                                 else {
2403                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2404                                 }
2405                                 M_CALL(REG_ITMP1);
2406                                 break;
2407
2408                         case ICMD_INVOKESPECIAL:
2409                                 emit_nullpointer_check(cd, iptr, REG_A0);
2410                                 /* fall through */
2411
2412                         case ICMD_INVOKESTATIC:
2413                                 if (lm == NULL) {
2414                                         disp = dseg_add_unique_address(cd, um);
2415
2416                                         patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
2417                                                                                   um, disp);
2418                                 }
2419                                 else {
2420                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2421                                 }
2422
2423                                 M_ALD(REG_ITMP2, RIP, disp);
2424                                 M_CALL(REG_ITMP2);
2425                                 break;
2426
2427                         case ICMD_INVOKEVIRTUAL:
2428                                 if (lm == NULL) {
2429                                         patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
2430
2431                                         s1 = 0;
2432                                 }
2433                                 else {
2434                                         s1 = OFFSET(vftbl_t, table[0]) +
2435                                                 sizeof(methodptr) * lm->vftblindex;
2436                                 }
2437
2438                                 /* implicit null-pointer check */
2439                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2440                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2441                                 M_CALL(REG_ITMP3);
2442                                 break;
2443
2444                         case ICMD_INVOKEINTERFACE:
2445                                 if (lm == NULL) {
2446                                         patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
2447
2448                                         s1 = 0;
2449                                         s2 = 0;
2450                                 }
2451                                 else {
2452                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2453                                                 sizeof(methodptr) * lm->class->index;
2454
2455                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2456                                 }
2457
2458                                 /* implicit null-pointer check */
2459                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2460                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2461                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2462                                 M_CALL(REG_ITMP3);
2463                                 break;
2464                         }
2465
2466                         /* generate method profiling code */
2467
2468                         PROFILE_CYCLE_START;
2469
2470                         /* store size of call code in replacement point */
2471
2472                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2473                         REPLACEMENT_POINT_FORGC_BUILTIN_RETURN(cd, iptr);
2474
2475                         /* store return value */
2476
2477                         switch (md->returntype.type) {
2478                         case TYPE_INT:
2479                         case TYPE_LNG:
2480                         case TYPE_ADR:
2481                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2482                                 M_INTMOVE(REG_RESULT, s1);
2483                                 emit_store_dst(jd, iptr, s1);
2484                                 break;
2485                         case TYPE_FLT:
2486                         case TYPE_DBL:
2487                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2488                                 M_FLTMOVE(REG_FRESULT, s1);
2489                                 emit_store_dst(jd, iptr, s1);
2490                                 break;
2491                         default:
2492                                 /* TYPE_VOID */
2493                                 break;
2494                         }
2495                         break;
2496
2497
2498                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2499
2500                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2501                                 /* object type cast-check */
2502
2503                                 classinfo *super;
2504                                 s4         superindex;
2505
2506                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2507                                         super      = NULL;
2508                                         superindex = 0;
2509                                 }
2510                                 else {
2511                                         super      = iptr->sx.s23.s3.c.cls;
2512                                         superindex = super->index;
2513                                 }
2514
2515                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2516                                         CODEGEN_CRITICAL_SECTION_NEW;
2517
2518                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2519
2520                                 /* if class is not resolved, check which code to call */
2521
2522                                 if (super == NULL) {
2523                                         M_TEST(s1);
2524                                         emit_label_beq(cd, BRANCH_LABEL_1);
2525
2526                                         patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2527                                                                                   iptr->sx.s23.s3.c.ref, 0);
2528
2529                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2530                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2531                                         emit_label_beq(cd, BRANCH_LABEL_2);
2532                                 }
2533
2534                                 /* interface checkcast code */
2535
2536                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2537                                         if (super != NULL) {
2538                                                 M_TEST(s1);
2539                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2540                                         }
2541
2542                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2543
2544                                         if (super == NULL) {
2545                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
2546                                                                                           iptr->sx.s23.s3.c.ref,
2547                                                                                           0);
2548                                         }
2549
2550                                         M_ILD32(REG_ITMP3,
2551                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2552                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2553                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2554
2555                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2556                                                         OFFSET(vftbl_t, interfacetable[0]) -
2557                                                         superindex * sizeof(methodptr*));
2558                                         M_TEST(REG_ITMP3);
2559                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2560
2561                                         if (super == NULL)
2562                                                 emit_label_br(cd, BRANCH_LABEL_4);
2563                                         else
2564                                                 emit_label(cd, BRANCH_LABEL_3);
2565                                 }
2566
2567                                 /* class checkcast code */
2568
2569                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2570                                         if (super == NULL) {
2571                                                 emit_label(cd, BRANCH_LABEL_2);
2572
2573                                                 cr   = iptr->sx.s23.s3.c.ref;
2574                                                 disp = dseg_add_unique_address(cd, cr);
2575
2576                                                 patcher_add_patch_ref(jd,
2577                                                                                           PATCHER_resolve_classref_to_vftbl,
2578                                                                                           cr, disp);
2579                                         }
2580                                         else {
2581                                                 M_TEST(s1);
2582                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2583
2584                                                 disp = dseg_add_address(cd, super->vftbl);
2585                                         }
2586
2587                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2588                                         M_ALD(REG_ITMP3, RIP, disp);
2589
2590                                         CODEGEN_CRITICAL_SECTION_START;
2591
2592                                         M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2593
2594                                         /*                                      if (s1 != REG_ITMP1) { */
2595                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2596                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2597                                         /*                                                                                              REG_ITMP1); */
2598                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2599                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2600                                         /*                                                                                              REG_ITMP3); */
2601                                         /*  #if defined(ENABLE_THREADS) */
2602                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2603                                         /*  #endif */
2604                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2605
2606                                         /*                                      } else { */
2607
2608                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
2609                                         M_ISUB(REG_ITMP3, REG_ITMP2);
2610                                         M_ALD(REG_ITMP3, RIP, disp);
2611                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2612                                         /*                                      } */
2613
2614                                         CODEGEN_CRITICAL_SECTION_END;
2615
2616                                         M_ICMP(REG_ITMP3, REG_ITMP2);
2617                                         emit_classcast_check(cd, iptr, BRANCH_UGT, REG_ITMP3, s1);
2618
2619                                         if (super != NULL)
2620                                                 emit_label(cd, BRANCH_LABEL_5);
2621                                 }
2622
2623                                 if (super == NULL) {
2624                                         emit_label(cd, BRANCH_LABEL_1);
2625                                         emit_label(cd, BRANCH_LABEL_4);
2626                                 }
2627
2628                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2629                         }
2630                         else {
2631                                 /* array type cast-check */
2632
2633                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2634                                 M_INTMOVE(s1, REG_A0);
2635
2636                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2637                                         cr   = iptr->sx.s23.s3.c.ref;
2638                                         disp = dseg_add_unique_address(cd, cr);
2639
2640                                         patcher_add_patch_ref(jd,
2641                                                                                   PATCHER_resolve_classref_to_classinfo,
2642                                                                                   cr, disp);
2643                                 }
2644                                 else {
2645                                         disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2646                                 }
2647
2648                                 M_ALD(REG_A1, RIP, disp);
2649                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2650                                 M_CALL(REG_ITMP1);
2651
2652                                 /* s1 may have been destroyed over the function call */
2653                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2654                                 M_TEST(REG_RESULT);
2655                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2656
2657                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2658                         }
2659
2660                         M_INTMOVE(s1, d);
2661                         emit_store_dst(jd, iptr, d);
2662                         break;
2663
2664                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2665
2666                         {
2667                         classinfo *super;
2668                         s4         superindex;
2669
2670                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2671                                 super      = NULL;
2672                                 superindex = 0;
2673                         }
2674                         else {
2675                                 super      = iptr->sx.s23.s3.c.cls;
2676                                 superindex = super->index;
2677                         }
2678
2679                         if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2680                                 CODEGEN_CRITICAL_SECTION_NEW;
2681
2682                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2683                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2684
2685                         if (s1 == d) {
2686                                 M_INTMOVE(s1, REG_ITMP1);
2687                                 s1 = REG_ITMP1;
2688                         }
2689
2690                         M_CLR(d);
2691
2692                         /* if class is not resolved, check which code to call */
2693
2694                         if (super == NULL) {
2695                                 M_TEST(s1);
2696                                 emit_label_beq(cd, BRANCH_LABEL_1);
2697
2698                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2699                                                                           iptr->sx.s23.s3.c.ref, 0);
2700
2701                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2702                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2703                                 emit_label_beq(cd, BRANCH_LABEL_2);
2704                         }
2705
2706                         /* interface instanceof code */
2707
2708                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2709                                 if (super != NULL) {
2710                                         M_TEST(s1);
2711                                         emit_label_beq(cd, BRANCH_LABEL_3);
2712                                 }
2713
2714                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2715
2716                                 if (super == NULL) {
2717                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
2718                                                                                   iptr->sx.s23.s3.c.ref, 0);
2719                                 }
2720
2721                                 M_ILD32(REG_ITMP3,
2722                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2723                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2724
2725                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2726
2727                                 M_BLE(a);
2728                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2729                                                 OFFSET(vftbl_t, interfacetable[0]) -
2730                                                 superindex * sizeof(methodptr*));
2731                                 M_TEST(REG_ITMP1);
2732                                 M_SETNE(d);
2733
2734                                 if (super == NULL)
2735                                         emit_label_br(cd, BRANCH_LABEL_4);
2736                                 else
2737                                         emit_label(cd, BRANCH_LABEL_3);
2738                         }
2739
2740                         /* class instanceof code */
2741
2742                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2743                                 if (super == NULL) {
2744                                         emit_label(cd, BRANCH_LABEL_2);
2745
2746                                         cr   = iptr->sx.s23.s3.c.ref;
2747                                         disp = dseg_add_unique_address(cd, cr);
2748
2749                                         patcher_add_patch_ref(jd,
2750                                                                                   PATCHER_resolve_classref_to_vftbl,
2751                                                                                   cr, disp);
2752                                 }
2753                                 else {
2754                                         M_TEST(s1);
2755                                         emit_label_beq(cd, BRANCH_LABEL_5);
2756
2757                                         disp = dseg_add_address(cd, super->vftbl);
2758                                 }
2759
2760                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2761                                 M_ALD(REG_ITMP2, RIP, disp);
2762
2763                                 CODEGEN_CRITICAL_SECTION_START;
2764
2765                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
2766                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2767                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2768
2769                                 CODEGEN_CRITICAL_SECTION_END;
2770
2771                                 M_ISUB(REG_ITMP2, REG_ITMP1);
2772                                 M_CLR(d); /* may be REG_ITMP2 */
2773                                 M_ICMP(REG_ITMP3, REG_ITMP1);
2774                                 M_SETULE(d);
2775
2776                                 if (super != NULL)
2777                                         emit_label(cd, BRANCH_LABEL_5);
2778                         }
2779
2780                         if (super == NULL) {
2781                                 emit_label(cd, BRANCH_LABEL_1);
2782                                 emit_label(cd, BRANCH_LABEL_4);
2783                         }
2784
2785                         emit_store_dst(jd, iptr, d);
2786                         }
2787                         break;
2788
2789                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2790
2791                         /* check for negative sizes and copy sizes to stack if necessary  */
2792
2793                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2794
2795                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2796
2797                                 /* copy SAVEDVAR sizes to stack */
2798                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2799
2800                                 /* Already Preallocated? */
2801                                 if (!(var->flags & PREALLOC)) {
2802                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2803                                         M_LST(s2, REG_SP, s1 * 8);
2804                                 }
2805                         }
2806
2807                         /* a0 = dimension count */
2808
2809                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2810
2811                         /* is a patcher function set? */
2812
2813                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2814                                 cr   = iptr->sx.s23.s3.c.ref;
2815                                 disp = dseg_add_unique_address(cd, cr);
2816
2817                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
2818                                                                           cr, disp);
2819                         }
2820                         else {
2821                                 disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2822                         }
2823
2824                         /* a1 = classinfo */
2825
2826                         M_ALD(REG_A1, RIP, disp);
2827
2828                         /* a2 = pointer to dimensions = stack pointer */
2829
2830                         M_MOV(REG_SP, REG_A2);
2831
2832                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2833                         M_CALL(REG_ITMP1);
2834
2835                         /* check for exception before result assignment */
2836
2837                         emit_exception_check(cd, iptr);
2838
2839                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2840                         M_INTMOVE(REG_RESULT, s1);
2841                         emit_store_dst(jd, iptr, s1);
2842                         break;
2843
2844                 default:
2845                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2846                                                                                    iptr->opc);
2847                         return false;
2848         } /* switch */
2849
2850         } /* for instruction */
2851
2852         MCODECHECK(512); /* XXX require a lower number? */
2853
2854         /* At the end of a basic block we may have to append some nops,
2855            because the patcher stub calling code might be longer than the
2856            actual instruction. So codepatching does not change the
2857            following block unintentionally. */
2858
2859         if (cd->mcodeptr < cd->lastmcodeptr) {
2860                 while (cd->mcodeptr < cd->lastmcodeptr) {
2861                         M_NOP;
2862                 }
2863         }
2864
2865         } /* if (bptr -> flags >= BBREACHED) */
2866         } /* for basic block */
2867
2868         dseg_createlinenumbertable(cd);
2869
2870         /* Generate patcher traps. */
2871
2872         emit_patcher_traps(jd);
2873
2874         /* everything's ok */
2875
2876         return true;
2877 }
2878
2879
2880 /* codegen_emit_stub_compiler **************************************************
2881
2882    Emit a stub routine which calls the compiler.
2883
2884 *******************************************************************************/
2885
2886 void codegen_emit_stub_compiler(jitdata *jd)
2887 {
2888         methodinfo  *m;
2889         codegendata *cd;
2890
2891         /* get required compiler data */
2892
2893         m  = jd->m;
2894         cd = jd->cd;
2895
2896         /* code for the stub */
2897
2898         M_LLD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));               /* methodinfo  */
2899 /*      M_ALD(REG_ITMP1, RIP, -(2 * SIZEOF_VOID_P));               /\* methodinfo  *\/ */
2900         M_LLD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));          /* compiler pointer */
2901 /*      M_ALD(REG_ITMP3, RIP, -(3 * SIZEOF_VOID_P));          /\* compiler pointer *\/ */
2902         M_JMP(REG_ITMP3);
2903 }
2904
2905
2906 /* codegen_emit_stub_native ****************************************************
2907
2908    Emits a stub routine which calls a native method.
2909
2910 *******************************************************************************/
2911
2912 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f, int skipparams)
2913 {
2914         methodinfo  *m;
2915         codeinfo    *code;
2916         codegendata *cd;
2917         methoddesc  *md;
2918         int          i, j;
2919         int          s1, s2;
2920         int          funcdisp;
2921 #if defined(ENABLE_GC_CACAO)
2922         int          disp;
2923 #endif
2924
2925         /* get required compiler data */
2926
2927         m    = jd->m;
2928         code = jd->code;
2929         cd   = jd->cd;
2930
2931         /* initialize variables */
2932
2933         md = m->parseddesc;
2934
2935         /* calculate stack frame size */
2936
2937         cd->stackframesize =
2938                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
2939                 sizeof(localref_table) / SIZEOF_VOID_P +
2940                 md->paramcount +
2941                 nmd->memuse;
2942
2943         ALIGN_ODD(cd->stackframesize);              /* keep stack 16-byte aligned */
2944
2945         /* create method header */
2946
2947         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2948         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2949         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
2950         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2951         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2952         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2953         (void) dseg_addlinenumbertablesize(cd);
2954         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
2955
2956 #if defined(ENABLE_PROFILING)
2957         /* generate native method profiling code */
2958
2959         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
2960                 /* count frequency */
2961
2962                 M_MOV_IMM(code, REG_ITMP3);
2963                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
2964         }
2965 #endif
2966
2967         /* generate stub code */
2968
2969         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2970
2971         /* get function address (this must happen before the stackframeinfo) */
2972
2973         funcdisp = dseg_add_functionptr(cd, f);
2974
2975         if (f == NULL)
2976                 patcher_add_patch_ref(jd, PATCHER_resolve_native_function, m, funcdisp);
2977
2978 #if defined(ENABLE_GC_CACAO)
2979         /* Save callee saved integer registers in stackframeinfo (GC may
2980            need to recover them during a collection). */
2981
2982         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
2983                 OFFSET(stackframeinfo, intregs);
2984
2985         for (i = 0; i < INT_SAV_CNT; i++)
2986                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2987 #endif
2988
2989         /* save integer and float argument registers */
2990
2991         for (i = 0; i < md->paramcount; i++) {
2992                 if (!md->params[i].inmemory) {
2993                         s1 = md->params[i].regoff;
2994
2995                         switch (md->paramtypes[i].type) {
2996                         case TYPE_INT:
2997                         case TYPE_LNG:
2998                         case TYPE_ADR:
2999                                 M_LST(s1, REG_SP, i * 8);
3000                                 break;
3001                         case TYPE_FLT:
3002                         case TYPE_DBL:
3003                                 M_DST(s1, REG_SP, i * 8);
3004                                 break;
3005                         }
3006                 }
3007         }
3008
3009         /* create dynamic stack info */
3010
3011         M_MOV(REG_SP, REG_A0);
3012         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3013         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3014         M_CALL(REG_ITMP1);
3015
3016         /* remember class argument */
3017
3018         if (m->flags & ACC_STATIC)
3019                 M_MOV(REG_RESULT, REG_ITMP2);
3020
3021         /* restore integer and float argument registers */
3022
3023         for (i = 0; i < md->paramcount; i++) {
3024                 if (!md->params[i].inmemory) {
3025                         s1 = md->params[i].regoff;
3026
3027                         switch (md->paramtypes[i].type) {
3028                         case TYPE_INT:
3029                         case TYPE_LNG:
3030                         case TYPE_ADR:
3031                                 M_LLD(s1, REG_SP, i * 8);
3032                                 break;
3033                         case TYPE_FLT:
3034                         case TYPE_DBL:
3035                                 M_DLD(s1, REG_SP, i * 8);
3036                                 break;
3037                         }
3038                 }
3039         }
3040
3041         /* Copy or spill arguments to new locations. */
3042
3043         for (i = md->paramcount - 1, j = i + skipparams; i >= 0; i--, j--) {
3044                 s2 = nmd->params[j].regoff;
3045
3046                 switch (md->paramtypes[i].type) {
3047                 case TYPE_INT:
3048                 case TYPE_LNG:
3049                 case TYPE_ADR:
3050                         if (!md->params[i].inmemory) {
3051                                 s1 = md->params[i].regoff;
3052
3053                                 if (!nmd->params[j].inmemory)
3054                                         M_INTMOVE(s1, s2);
3055                                 else
3056                                         M_LST(s1, REG_SP, s2);
3057                         }
3058                         else {
3059                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3060                                 M_LLD(REG_ITMP1, REG_SP, s1);
3061                                 M_LST(REG_ITMP1, REG_SP, s2);
3062                         }
3063                         break;
3064                 case TYPE_FLT:
3065                         /* We only copy spilled float arguments, as the float
3066                            argument registers keep unchanged. */
3067
3068                         if (md->params[i].inmemory) {
3069                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3070
3071                                 M_FLD(REG_FTMP1, REG_SP, s1);
3072                                 M_FST(REG_FTMP1, REG_SP, s2);
3073                         }
3074                         break;
3075                 case TYPE_DBL:
3076                         if (md->params[i].inmemory) {
3077                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3078                                 M_DLD(REG_FTMP1, REG_SP, s1);
3079                                 M_DST(REG_FTMP1, REG_SP, s2);
3080                         }
3081                         break;
3082                 }
3083         }
3084
3085         /* Handle native Java methods. */
3086
3087         if (m->flags & ACC_NATIVE) {
3088                 /* put class into second argument register */
3089
3090                 if (m->flags & ACC_STATIC)
3091                         M_MOV(REG_ITMP2, REG_A1);
3092
3093                 /* put env into first argument register */
3094
3095                 M_MOV_IMM(_Jv_env, REG_A0);
3096         }
3097
3098         /* do the native function call */
3099
3100         M_ALD(REG_ITMP1, RIP, funcdisp);
3101         M_CALL(REG_ITMP1);
3102
3103         /* save return value */
3104
3105         switch (md->returntype.type) {
3106         case TYPE_INT:
3107         case TYPE_LNG:
3108         case TYPE_ADR:
3109                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3110                 break;
3111         case TYPE_FLT:
3112         case TYPE_DBL:
3113                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3114                 break;
3115         case TYPE_VOID:
3116                 break;
3117         }
3118
3119         /* remove native stackframe info */
3120
3121         M_MOV(REG_SP, REG_A0);
3122         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3123         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3124         M_CALL(REG_ITMP1);
3125         M_MOV(REG_RESULT, REG_ITMP3);
3126
3127         /* restore return value */
3128
3129         switch (md->returntype.type) {
3130         case TYPE_INT:
3131         case TYPE_LNG:
3132         case TYPE_ADR:
3133                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3134                 break;
3135         case TYPE_FLT:
3136         case TYPE_DBL:
3137                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3138                 break;
3139         case TYPE_VOID:
3140                 break;
3141         }
3142
3143 #if defined(ENABLE_GC_CACAO)
3144         /* Restore callee saved integer registers from stackframeinfo (GC
3145            might have modified them during a collection). */
3146
3147         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3148                 OFFSET(stackframeinfo, intregs);
3149
3150         for (i = 0; i < INT_SAV_CNT; i++)
3151                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3152 #endif
3153
3154         /* remove stackframe */
3155
3156         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3157
3158         /* test for exception */
3159
3160         M_TEST(REG_ITMP3);
3161         M_BNE(1);
3162         M_RET;
3163
3164         /* handle exception */
3165
3166         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3167         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3168         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3169
3170         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3171         M_JMP(REG_ITMP3);
3172
3173         /* Generate patcher traps. */
3174
3175         emit_patcher_traps(jd);
3176 }
3177
3178
3179 /*
3180  * These are local overrides for various environment variables in Emacs.
3181  * Please do not remove this and leave it at the end of the file, where
3182  * Emacs will automagically detect them.
3183  * ---------------------------------------------------------------------
3184  * Local variables:
3185  * mode: c
3186  * indent-tabs-mode: t
3187  * c-basic-offset: 4
3188  * tab-width: 4
3189  * End:
3190  * vim:noexpandtab:sw=4:ts=4:
3191  */