src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    $Id: codegen.c 7596 2007-03-28 21:05:53Z twisti $
  26
  27 */
  28
  29
  30 #include "config.h"
  31
  32 #include <assert.h>
  33 #include <stdio.h>
  34
  35 #include "vm/types.h"
  36
  37 #include "md-abi.h"
  38
  39 #include "vm/jit/x86_64/arch.h"
  40 #include "vm/jit/x86_64/codegen.h"
  41 #include "vm/jit/x86_64/emit.h"
  42
  43 #include "mm/memory.h"
  44 #include "native/jni.h"
  45 #include "native/native.h"
  46
  47 #if defined(ENABLE_THREADS)
  48 # include "threads/native/lock.h"
  49 #endif
  50
  51 #include "vm/builtin.h"
  52 #include "vm/exceptions.h"
  53 #include "vm/global.h"
  54 #include "vm/stringlocal.h"
  55 #include "vm/vm.h"
  56
  57 #include "vm/jit/asmpart.h"
  58 #include "vm/jit/codegen-common.h"
  59 #include "vm/jit/dseg.h"
  60 #include "vm/jit/emit-common.h"
  61 #include "vm/jit/jit.h"
  62 #include "vm/jit/methodheader.h"
  63 #include "vm/jit/parse.h"
  64 #include "vm/jit/patcher.h"
  65 #include "vm/jit/reg.h"
  66 #include "vm/jit/replace.h"
  67 #include "vm/jit/stacktrace.h"
  68
  69 #if defined(ENABLE_LSRA)
  70 # include "vm/jit/allocator/lsra.h"
  71 #endif
  72
  73 #include "vmcore/loader.h"
  74 #include "vmcore/options.h"
  75 #include "vmcore/statistics.h"
  76
  77
  78 /* codegen_emit ****************************************************************
  79
  80    Generates machine code.
  81
  82 *******************************************************************************/
  83
  84 bool codegen_emit(jitdata *jd)
  85 {
  86         methodinfo         *m;
  87         codeinfo           *code;
  88         codegendata        *cd;
  89         registerdata       *rd;
  90         s4                  len, s1, s2, s3, d, disp;
  91         u2                  currentline;
  92         ptrint              a;
  93         varinfo            *var, *dst;
  94         basicblock         *bptr;
  95         instruction        *iptr;
  96         exception_entry    *ex;
  97         constant_classref  *cr;
  98         unresolved_class   *uc;
  99         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 100         unresolved_method  *um;
 101         builtintable_entry *bte;
 102         methoddesc         *md;
 103         fieldinfo          *fi;
 104         unresolved_field   *uf;
 105         s4                  fieldtype;
 106         s4                 varindex;
 107
 108         /* get required compiler data */
 109
 110         m    = jd->m;
 111         code = jd->code;
 112         cd   = jd->cd;
 113         rd   = jd->rd;
 114
 115         /* prevent compiler warnings */
 116
 117         d   = 0;
 118         lm  = NULL;
 119         um  = NULL;
 120         bte = NULL;
 121
 122         {
 123         s4 i, p, t, l;
 124         s4 savedregs_num;
 125
 126         savedregs_num = 0;
 127
 128         /* space to save used callee saved registers */
 129
 130         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 131         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 132
 133         cd->stackframesize = rd->memuse + savedregs_num;
 134
 135 #if defined(ENABLE_THREADS)
 136         /* space to save argument of monitor_enter */
 137
 138         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 139                 cd->stackframesize++;
 140 #endif
 141
 142         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 143            native code e.g. libc or jni (alignment problems with
 144            movaps). */
 145
 146         if (!jd->isleafmethod || opt_verbosecall)
 147                 cd->stackframesize |= 0x1;
 148
 149         /* create method header */
 150
 151         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 152         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 153
 154 #if defined(ENABLE_THREADS)
 155         /* IsSync contains the offset relative to the stack pointer for the
 156            argument of monitor_exit used in the exception handler. Since the
 157            offset could be zero and give a wrong meaning of the flag it is
 158            offset by one.
 159         */
 160
 161         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 162                 (void) dseg_add_unique_s4(cd, (rd->memuse + 1) * 8); /* IsSync        */
 163         else
 164 #endif
 165                 (void) dseg_add_unique_s4(cd, 0);                  /* IsSync          */
 166
 167         (void) dseg_add_unique_s4(cd, jd->isleafmethod);               /* IsLeaf  */
 168         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 169         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 170
 171         (void) dseg_addlinenumbertablesize(cd);
 172
 173         (void) dseg_add_unique_s4(cd, jd->exceptiontablelength); /* ExTableSize   */
 174
 175         /* create exception table */
 176
 177         for (ex = jd->exceptiontable; ex != NULL; ex = ex->down) {
 178                 dseg_add_target(cd, ex->start);
 179                 dseg_add_target(cd, ex->end);
 180                 dseg_add_target(cd, ex->handler);
 181                 (void) dseg_add_unique_address(cd, ex->catchtype.any);
 182         }
 183
 184 #if defined(ENABLE_PROFILING)
 185         /* generate method profiling code */
 186
 187         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 188                 /* count frequency */
 189
 190                 M_MOV_IMM(code, REG_ITMP3);
 191                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 192
 193                 PROFILE_CYCLE_START;
 194         }
 195 #endif
 196
 197         /* create stack frame (if necessary) */
 198
 199         if (cd->stackframesize)
 200                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 201
 202         /* save used callee saved registers */
 203
 204         p = cd->stackframesize;
 205         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 206                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 207         }
 208         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 209                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 210         }
 211
 212         /* take arguments out of register or stack frame */
 213
 214         md = m->parseddesc;
 215
 216         for (p = 0, l = 0; p < md->paramcount; p++) {
 217                 t = md->paramtypes[p].type;
 218
 219                 varindex = jd->local_map[l * 5 + t];
 220
 221                 l++;
 222                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 223                         l++;
 224
 225                 if (varindex == UNUSED)
 226                         continue;
 227
 228                 var = VAR(varindex);
 229
 230                 s1 = md->params[p].regoff;
 231
 232                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 233                         s2 = rd->argintregs[s1];
 234                         if (!md->params[p].inmemory) {           /* register arguments    */
 235                                 if (!IS_INMEMORY(var->flags)) {      /* reg arg -> register   */
 236                                         M_INTMOVE(s2, var->vv.regoff);
 237
 238                                 } else {                             /* reg arg -> spilled    */
 239                                     M_LST(s2, REG_SP, var->vv.regoff * 8);
 240                                 }
 241
 242                         } else {                                 /* stack arguments       */
 243                                 if (!IS_INMEMORY(var->flags)) {      /* stack arg -> register */
 244                                         /* + 8 for return address */
 245                                         M_LLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 246
 247                                 } else {                             /* stack arg -> spilled  */
 248                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 249                                 }
 250                         }
 251
 252                 } else {                                     /* floating args         */
 253                         if (!md->params[p].inmemory) {           /* register arguments    */
 254                                 s2 = rd->argfltregs[s1];
 255                                 if (!IS_INMEMORY(var->flags)) {      /* reg arg -> register   */
 256                                         M_FLTMOVE(s2, var->vv.regoff);
 257
 258                                 } else {                                         /* reg arg -> spilled    */
 259                                         M_DST(s2, REG_SP, var->vv.regoff * 8);
 260                                 }
 261
 262                         } else {                                 /* stack arguments       */
 263                                 if (!IS_INMEMORY(var->flags)) {      /* stack-arg -> register */
 264                                         M_DLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 265
 266                                 } else {
 267                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 268                                 }
 269                         }
 270                 }
 271         }  /* end for */
 272
 273         /* save monitorenter argument */
 274
 275 #if defined(ENABLE_THREADS)
 276         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 277                 /* stack offset for monitor argument */
 278
 279                 s1 = rd->memuse;
 280
 281                 if (opt_verbosecall) {
 282                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 283
 284                         for (p = 0; p < INT_ARG_CNT; p++)
 285                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 286
 287                         for (p = 0; p < FLT_ARG_CNT; p++)
 288                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 289
 290                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 291                 }
 292
 293                 /* decide which monitor enter function to call */
 294
 295                 if (m->flags & ACC_STATIC) {
 296                         M_MOV_IMM(&m->class->object.header, REG_A0);
 297                 }
 298                 else {
 299                         M_TEST(REG_A0);
 300                         M_BNE(8);
 301                         M_ALD_MEM(REG_A0, EXCEPTION_HARDWARE_NULLPOINTER);
 302                 }
 303
 304                 M_AST(REG_A0, REG_SP, s1 * 8);
 305                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 306                 M_CALL(REG_ITMP1);
 307
 308                 if (opt_verbosecall) {
 309                         for (p = 0; p < INT_ARG_CNT; p++)
 310                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 311
 312                         for (p = 0; p < FLT_ARG_CNT; p++)
 313                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 314
 315                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 316                 }
 317         }
 318 #endif
 319
 320 #if !defined(NDEBUG)
 321         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 322                 emit_verbosecall_enter(jd);
 323 #endif /* !defined(NDEBUG) */
 324
 325         }
 326
 327         /* end of header generation */
 328
 329         /* create replacement points */
 330
 331         REPLACEMENT_POINTS_INIT(cd, jd);
 332
 333         /* walk through all basic blocks */
 334
 335         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 336
 337                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 338
 339                 if (bptr->flags >= BBREACHED) {
 340
 341                 /* branch resolving */
 342
 343                 codegen_resolve_branchrefs(cd, bptr);
 344
 345                 /* handle replacement points */
 346
 347                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 348
 349                 /* copy interface registers to their destination */
 350
 351                 len = bptr->indepth;
 352                 MCODECHECK(512);
 353
 354 #if defined(ENABLE_PROFILING)
 355                 /* generate basicblock profiling code */
 356
 357                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 358                         /* count frequency */
 359
 360                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 361                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 362
 363                         /* if this is an exception handler, start profiling again */
 364
 365                         if (bptr->type == BBTYPE_EXH)
 366                                 PROFILE_CYCLE_START;
 367                 }
 368 #endif
 369
 370 #if defined(ENABLE_LSRA)
 371                 if (opt_lsra) {
 372                         while (len) {
 373                                 len--;
 374                                 src = bptr->invars[len];
 375                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 376                                         if (bptr->type == BBTYPE_EXH) {
 377 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 378                                                 if (!IS_INMEMORY(src->flags))
 379                                                         d= src->vv.regoff;
 380                                                 else
 381                                                         d=REG_ITMP1;
 382                                                 M_INTMOVE(REG_ITMP1, d);
 383                                                 emit_store(jd, NULL, src, d);
 384                                         }
 385                                 }
 386                         }
 387
 388                 } else {
 389 #endif
 390
 391                 while (len) {
 392                         len--;
 393                         var = VAR(bptr->invars[len]);
 394                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 395                                 if (bptr->type == BBTYPE_EXH) {
 396                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 397                                         M_INTMOVE(REG_ITMP1, d);
 398                                         emit_store(jd, NULL, var, d);
 399                                 }
 400                         }
 401                         else {
 402                                 assert((var->flags & INOUT));
 403                         }
 404                 }
 405 #if defined(ENABLE_LSRA)
 406                 }
 407 #endif
 408                 /* walk through all instructions */
 409
 410                 len = bptr->icount;
 411                 currentline = 0;
 412
 413                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 414                         if (iptr->line != currentline) {
 415                                 dseg_addlinenumber(cd, iptr->line);
 416                                 currentline = iptr->line;
 417                         }
 418
 419                         MCODECHECK(1024);                         /* 1KB should be enough */
 420
 421                 switch (iptr->opc) {
 422                 case ICMD_NOP:        /* ...  ==> ...                                 */
 423                 case ICMD_POP:        /* ..., value  ==> ...                          */
 424                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 425                         break;
 426
 427                 case ICMD_INLINE_START:
 428
 429                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 430                         break;
 431
 432                 case ICMD_INLINE_BODY:
 433
 434                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 435                         dseg_addlinenumber_inline_start(cd, iptr);
 436                         dseg_addlinenumber(cd, iptr->line);
 437                         break;
 438
 439                 case ICMD_INLINE_END:
 440
 441                         dseg_addlinenumber_inline_end(cd, iptr);
 442                         dseg_addlinenumber(cd, iptr->line);
 443                         break;
 444
 445                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 446
 447                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 448                         emit_nullpointer_check(cd, iptr, s1);
 449                         break;
 450
 451                 /* constant operations ************************************************/
 452
 453                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 454
 455                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 456                         ICONST(d, iptr->sx.val.i);
 457                         emit_store_dst(jd, iptr, d);
 458                         break;
 459
 460                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 461
 462                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 463                         LCONST(d, iptr->sx.val.l);
 464                         emit_store_dst(jd, iptr, d);
 465                         break;
 466
 467                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 468
 469                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 470                         disp = dseg_add_float(cd, iptr->sx.val.f);
 471                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 472                         emit_store_dst(jd, iptr, d);
 473                         break;
 474
 475                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 476
 477                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 478                         disp = dseg_add_double(cd, iptr->sx.val.d);
 479                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 480                         emit_store_dst(jd, iptr, d);
 481                         break;
 482
 483                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 484
 485                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 486
 487                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 488                                 cr = iptr->sx.val.c.ref;
 489
 490 /*                              PROFILE_CYCLE_STOP; */
 491
 492                                 codegen_add_patch_ref(cd, PATCHER_aconst, cr, 0);
 493
 494 /*                              PROFILE_CYCLE_START; */
 495
 496                                 M_MOV_IMM(NULL, d);
 497
 498                         } else {
 499                                 if (iptr->sx.val.anyptr == 0)
 500                                         M_CLR(d);
 501                                 else
 502                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 503                         }
 504                         emit_store_dst(jd, iptr, d);
 505                         break;
 506
 507
 508                 /* load/store/copy/move operations ************************************/
 509
 510                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 511                 case ICMD_ALOAD:      /* s1 = local variable                          */
 512                 case ICMD_LLOAD:
 513                 case ICMD_FLOAD:
 514                 case ICMD_DLOAD:
 515                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 516                 case ICMD_LSTORE:
 517                 case ICMD_FSTORE:
 518                 case ICMD_DSTORE:
 519                 case ICMD_COPY:
 520                 case ICMD_MOVE:
 521
 522                         emit_copy(jd, iptr, VAROP(iptr->s1), VAROP(iptr->dst));
 523                         break;
 524
 525                 case ICMD_ASTORE:
 526                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 527                                 emit_copy(jd, iptr, VAROP(iptr->s1), VAROP(iptr->dst));
 528                         break;
 529
 530                 /* integer operations *************************************************/
 531
 532                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 533
 534                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 535                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 536                         M_INTMOVE(s1, d);
 537                         M_INEG(d);
 538                         emit_store_dst(jd, iptr, d);
 539                         break;
 540
 541                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 542
 543                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 544                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 545                         M_INTMOVE(s1, d);
 546                         M_LNEG(d);
 547                         emit_store_dst(jd, iptr, d);
 548                         break;
 549
 550                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 551
 552                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 553                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 554                         M_ISEXT(s1, d);
 555                         emit_store_dst(jd, iptr, d);
 556                         break;
 557
 558                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 559
 560                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 561                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 562                         M_IMOV(s1, d);
 563                         emit_store_dst(jd, iptr, d);
 564                         break;
 565
 566                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 567
 568                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 569                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 570                         M_BSEXT(s1, d);
 571                         emit_store_dst(jd, iptr, d);
 572                         break;
 573
 574                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 575
 576                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 577                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 578                         M_CZEXT(s1, d);
 579                         emit_store_dst(jd, iptr, d);
 580                         break;
 581
 582                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 583
 584                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 585                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 586                         M_SSEXT(s1, d);
 587                         emit_store_dst(jd, iptr, d);
 588                         break;
 589
 590
 591                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 592
 593                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 594                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 595                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 596                         if (s2 == d)
 597                                 M_IADD(s1, d);
 598                         else {
 599                                 M_INTMOVE(s1, d);
 600                                 M_IADD(s2, d);
 601                         }
 602                         emit_store_dst(jd, iptr, d);
 603                         break;
 604
 605                 case ICMD_IINC:
 606                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 607                                       /* sx.val.i = constant                             */
 608
 609                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 610                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 611
 612                         /* Using inc and dec is not faster than add (tested with
 613                            sieve). */
 614
 615                         M_INTMOVE(s1, d);
 616                         M_IADD_IMM(iptr->sx.val.i, d);
 617                         emit_store_dst(jd, iptr, d);
 618                         break;
 619
 620                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 621
 622                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 623                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 624                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 625                         if (s2 == d)
 626                                 M_LADD(s1, d);
 627                         else {
 628                                 M_INTMOVE(s1, d);
 629                                 M_LADD(s2, d);
 630                         }
 631                         emit_store_dst(jd, iptr, d);
 632                         break;
 633
 634                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 635                                       /* sx.val.l = constant                             */
 636
 637                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 638                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 639                         M_INTMOVE(s1, d);
 640                         if (IS_IMM32(iptr->sx.val.l))
 641                                 M_LADD_IMM(iptr->sx.val.l, d);
 642                         else {
 643                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 644                                 M_LADD(REG_ITMP2, d);
 645                         }
 646                         emit_store_dst(jd, iptr, d);
 647                         break;
 648
 649                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 650
 651                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 652                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 653                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 654                         if (s2 == d) {
 655                                 M_INTMOVE(s1, REG_ITMP1);
 656                                 M_ISUB(s2, REG_ITMP1);
 657                                 M_INTMOVE(REG_ITMP1, d);
 658                         } else {
 659                                 M_INTMOVE(s1, d);
 660                                 M_ISUB(s2, d);
 661                         }
 662                         emit_store_dst(jd, iptr, d);
 663                         break;
 664
 665                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 666                                       /* sx.val.i = constant                             */
 667
 668                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 669                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 670                         M_INTMOVE(s1, d);
 671                         M_ISUB_IMM(iptr->sx.val.i, d);
 672                         emit_store_dst(jd, iptr, d);
 673                         break;
 674
 675                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 676
 677                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 678                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 679                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 680                         if (s2 == d) {
 681                                 M_INTMOVE(s1, REG_ITMP1);
 682                                 M_LSUB(s2, REG_ITMP1);
 683                                 M_INTMOVE(REG_ITMP1, d);
 684                         } else {
 685                                 M_INTMOVE(s1, d);
 686                                 M_LSUB(s2, d);
 687                         }
 688                         emit_store_dst(jd, iptr, d);
 689                         break;
 690
 691                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 692                                       /* sx.val.l = constant                             */
 693
 694                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 695                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 696                         M_INTMOVE(s1, d);
 697                         if (IS_IMM32(iptr->sx.val.l))
 698                                 M_LSUB_IMM(iptr->sx.val.l, d);
 699                         else {
 700                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 701                                 M_LSUB(REG_ITMP2, d);
 702                         }
 703                         emit_store_dst(jd, iptr, d);
 704                         break;
 705
 706                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 707
 708                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 709                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 710                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 711                         if (s2 == d)
 712                                 M_IMUL(s1, d);
 713                         else {
 714                                 M_INTMOVE(s1, d);
 715                                 M_IMUL(s2, d);
 716                         }
 717                         emit_store_dst(jd, iptr, d);
 718                         break;
 719
 720                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 721                                       /* sx.val.i = constant                             */
 722
 723                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 724                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 725                         if (iptr->sx.val.i == 2) {
 726                                 M_INTMOVE(s1, d);
 727                                 M_ISLL_IMM(1, d);
 728                         } else
 729                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 730                         emit_store_dst(jd, iptr, d);
 731                         break;
 732
 733                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 734
 735                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 736                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 737                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 738                         if (s2 == d)
 739                                 M_LMUL(s1, d);
 740                         else {
 741                                 M_INTMOVE(s1, d);
 742                                 M_LMUL(s2, d);
 743                         }
 744                         emit_store_dst(jd, iptr, d);
 745                         break;
 746
 747                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 748                                       /* sx.val.l = constant                             */
 749
 750                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 751                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 752                         if (IS_IMM32(iptr->sx.val.l))
 753                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 754                         else {
 755                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 756                                 M_INTMOVE(s1, d);
 757                                 M_LMUL(REG_ITMP2, d);
 758                         }
 759                         emit_store_dst(jd, iptr, d);
 760                         break;
 761
 762                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 763
 764                         s1 = emit_load_s1(jd, iptr, RAX);
 765                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 766                         d = codegen_reg_of_dst(jd, iptr, RAX);
 767
 768                         M_INTMOVE(s1, RAX);
 769                         M_INTMOVE(s2, REG_ITMP3);
 770                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 771
 772                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 773
 774                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 775                         M_BNE(4 + 6);
 776                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 777                         M_BEQ(1 + 3);                                          /* 6 bytes */
 778
 779                         emit_cltd(cd);                                         /* 1 byte  */
 780                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 781
 782                         M_INTMOVE(RAX, d);
 783                         emit_store_dst(jd, iptr, d);
 784                         dst = VAROP(iptr->dst);
 785                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 786                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 787                         break;
 788
 789                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 790
 791                         s1 = emit_load_s1(jd, iptr, RAX);
 792                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 793                         d = codegen_reg_of_dst(jd, iptr, RDX);
 794
 795                         M_INTMOVE(s1, RAX);
 796                         M_INTMOVE(s2, REG_ITMP3);
 797                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 798
 799                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 800
 801                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 802                         M_BNE(3 + 4 + 6);
 803                         M_CLR(RDX);                                            /* 3 bytes */
 804                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 805                         M_BEQ(1 + 3);                                          /* 6 bytes */
 806
 807                         emit_cltd(cd);                                         /* 1 byte  */
 808                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 809
 810                         M_INTMOVE(RDX, d);
 811                         emit_store_dst(jd, iptr, d);
 812                         dst = VAROP(iptr->dst);
 813                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 814                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 815                         break;
 816
 817                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 818                                       /* sx.val.i = constant                             */
 819
 820                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 821                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 822                         M_INTMOVE(s1, REG_ITMP1);
 823                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 824                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 825                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 826                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 827                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 828                         emit_store_dst(jd, iptr, d);
 829                         break;
 830
 831                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 832                                       /* sx.val.i = constant                             */
 833
 834                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 835                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 836                         M_INTMOVE(s1, REG_ITMP1);
 837                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 838                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 839                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 840                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 841                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 842                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 843                         emit_store_dst(jd, iptr, d);
 844                         break;
 845
 846
 847                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 848
 849                         s1 = emit_load_s1(jd, iptr, RAX);
 850                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 851                         d = codegen_reg_of_dst(jd, iptr, RAX);
 852
 853                         M_INTMOVE(s1, RAX);
 854                         M_INTMOVE(s2, REG_ITMP3);
 855                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 856
 857                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 858
 859                         /* check as described in jvm spec */
 860                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 861                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 862                         M_BNE(4 + 6);
 863                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 864                         M_BEQ(2 + 3);                                          /* 6 bytes */
 865
 866                         emit_cqto(cd);                                         /* 2 bytes */
 867                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 868
 869                         M_INTMOVE(RAX, d);
 870                         emit_store_dst(jd, iptr, d);
 871                         dst = VAROP(iptr->dst);
 872                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 873                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 874                         break;
 875
 876                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 877
 878                         s1 = emit_load_s1(jd, iptr, RAX);
 879                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 880                         d = codegen_reg_of_dst(jd, iptr, RDX);
 881
 882                         M_INTMOVE(s1, RAX);
 883                         M_INTMOVE(s2, REG_ITMP3);
 884                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 885
 886                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 887
 888                         /* check as described in jvm spec */
 889                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 890                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 891                         M_BNE(3 + 4 + 6);
 892                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 893                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 894                         M_BEQ(2 + 3);                                          /* 6 bytes */
 895
 896                         emit_cqto(cd);                                         /* 2 bytes */
 897                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 898
 899                         M_INTMOVE(RDX, d);
 900                         emit_store_dst(jd, iptr, d);
 901                         dst = VAROP(iptr->dst);
 902                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 903                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 904                         break;
 905
 906                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 907                                       /* sx.val.i = constant                             */
 908
 909                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 910                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 911                         M_INTMOVE(s1, REG_ITMP1);
 912                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 913                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 914                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 915                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 916                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 917                         emit_store_dst(jd, iptr, d);
 918                         break;
 919
 920                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 921                                       /* sx.val.l = constant                             */
 922
 923                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 924                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 925                         M_INTMOVE(s1, REG_ITMP1);
 926                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 927                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 928                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 929                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 930                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 931                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 932                         emit_store_dst(jd, iptr, d);
 933                         break;
 934
 935                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 936
 937                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 938                         emit_ishift(jd, SHIFT_SHL, iptr);
 939                         break;
 940
 941                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 942                                       /* sx.val.i = constant                             */
 943
 944                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 945                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 946                         M_INTMOVE(s1, d);
 947                         M_ISLL_IMM(iptr->sx.val.i, d);
 948                         emit_store_dst(jd, iptr, d);
 949                         break;
 950
 951                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 952
 953                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 954                         emit_ishift(jd, SHIFT_SAR, iptr);
 955                         break;
 956
 957                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 958                                       /* sx.val.i = constant                             */
 959
 960                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 961                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 962                         M_INTMOVE(s1, d);
 963                         M_ISRA_IMM(iptr->sx.val.i, d);
 964                         emit_store_dst(jd, iptr, d);
 965                         break;
 966
 967                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 968
 969                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 970                         emit_ishift(jd, SHIFT_SHR, iptr);
 971                         break;
 972
 973                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 974                                       /* sx.val.i = constant                             */
 975
 976                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 977                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 978                         M_INTMOVE(s1, d);
 979                         M_ISRL_IMM(iptr->sx.val.i, d);
 980                         emit_store_dst(jd, iptr, d);
 981                         break;
 982
 983                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 984
 985                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 986                         emit_lshift(jd, SHIFT_SHL, iptr);
 987                         break;
 988
 989         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 990                                           /* sx.val.i = constant                             */
 991
 992                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 993                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 994                         M_INTMOVE(s1, d);
 995                         M_LSLL_IMM(iptr->sx.val.i, d);
 996                         emit_store_dst(jd, iptr, d);
 997                         break;
 998
 999                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1000
1001                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1002                         emit_lshift(jd, SHIFT_SAR, iptr);
1003                         break;
1004
1005                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1006                                       /* sx.val.i = constant                             */
1007
1008                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1009                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1010                         M_INTMOVE(s1, d);
1011                         M_LSRA_IMM(iptr->sx.val.i, d);
1012                         emit_store_dst(jd, iptr, d);
1013                         break;
1014
1015                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1016
1017                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1018                         emit_lshift(jd, SHIFT_SHR, iptr);
1019                         break;
1020
1021                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1022                                       /* sx.val.l = constant                             */
1023
1024                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1025                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1026                         M_INTMOVE(s1, d);
1027                         M_LSRL_IMM(iptr->sx.val.i, d);
1028                         emit_store_dst(jd, iptr, d);
1029                         break;
1030
1031                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1032
1033                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1034                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1035                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1036                         if (s2 == d)
1037                                 M_IAND(s1, d);
1038                         else {
1039                                 M_INTMOVE(s1, d);
1040                                 M_IAND(s2, d);
1041                         }
1042                         emit_store_dst(jd, iptr, d);
1043                         break;
1044
1045                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1046                                       /* sx.val.i = constant                             */
1047
1048                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1049                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1050                         M_INTMOVE(s1, d);
1051                         M_IAND_IMM(iptr->sx.val.i, d);
1052                         emit_store_dst(jd, iptr, d);
1053                         break;
1054
1055                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1056
1057                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1058                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1059                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1060                         if (s2 == d)
1061                                 M_LAND(s1, d);
1062                         else {
1063                                 M_INTMOVE(s1, d);
1064                                 M_LAND(s2, d);
1065                         }
1066                         emit_store_dst(jd, iptr, d);
1067                         break;
1068
1069                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1070                                       /* sx.val.l = constant                             */
1071
1072                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1073                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1074                         M_INTMOVE(s1, d);
1075                         if (IS_IMM32(iptr->sx.val.l))
1076                                 M_LAND_IMM(iptr->sx.val.l, d);
1077                         else {
1078                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1079                                 M_LAND(REG_ITMP2, d);
1080                         }
1081                         emit_store_dst(jd, iptr, d);
1082                         break;
1083
1084                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1085
1086                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1087                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1088                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1089                         if (s2 == d)
1090                                 M_IOR(s1, d);
1091                         else {
1092                                 M_INTMOVE(s1, d);
1093                                 M_IOR(s2, d);
1094                         }
1095                         emit_store_dst(jd, iptr, d);
1096                         break;
1097
1098                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1099                                       /* sx.val.i = constant                             */
1100
1101                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1102                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1103                         M_INTMOVE(s1, d);
1104                         M_IOR_IMM(iptr->sx.val.i, d);
1105                         emit_store_dst(jd, iptr, d);
1106                         break;
1107
1108                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1109
1110                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1111                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1112                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1113                         if (s2 == d)
1114                                 M_LOR(s1, d);
1115                         else {
1116                                 M_INTMOVE(s1, d);
1117                                 M_LOR(s2, d);
1118                         }
1119                         emit_store_dst(jd, iptr, d);
1120                         break;
1121
1122                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1123                                       /* sx.val.l = constant                             */
1124
1125                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1126                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1127                         M_INTMOVE(s1, d);
1128                         if (IS_IMM32(iptr->sx.val.l))
1129                                 M_LOR_IMM(iptr->sx.val.l, d);
1130                         else {
1131                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1132                                 M_LOR(REG_ITMP2, d);
1133                         }
1134                         emit_store_dst(jd, iptr, d);
1135                         break;
1136
1137                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1138
1139                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1140                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1141                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1142                         if (s2 == d)
1143                                 M_IXOR(s1, d);
1144                         else {
1145                                 M_INTMOVE(s1, d);
1146                                 M_IXOR(s2, d);
1147                         }
1148                         emit_store_dst(jd, iptr, d);
1149                         break;
1150
1151                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1152                                       /* sx.val.i = constant                             */
1153
1154                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1155                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1156                         M_INTMOVE(s1, d);
1157                         M_IXOR_IMM(iptr->sx.val.i, d);
1158                         emit_store_dst(jd, iptr, d);
1159                         break;
1160
1161                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1162
1163                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1164                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1165                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1166                         if (s2 == d)
1167                                 M_LXOR(s1, d);
1168                         else {
1169                                 M_INTMOVE(s1, d);
1170                                 M_LXOR(s2, d);
1171                         }
1172                         emit_store_dst(jd, iptr, d);
1173                         break;
1174
1175                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1176                                       /* sx.val.l = constant                             */
1177
1178                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1179                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1180                         M_INTMOVE(s1, d);
1181                         if (IS_IMM32(iptr->sx.val.l))
1182                                 M_LXOR_IMM(iptr->sx.val.l, d);
1183                         else {
1184                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1185                                 M_LXOR(REG_ITMP2, d);
1186                         }
1187                         emit_store_dst(jd, iptr, d);
1188                         break;
1189
1190
1191                 /* floating operations ************************************************/
1192
1193                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1194
1195                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1196                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1197                         disp = dseg_add_s4(cd, 0x80000000);
1198                         M_FLTMOVE(s1, d);
1199                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1200                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1201                         emit_store_dst(jd, iptr, d);
1202                         break;
1203
1204                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1205
1206                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1207                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1208                         disp = dseg_add_s8(cd, 0x8000000000000000);
1209                         M_FLTMOVE(s1, d);
1210                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1211                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1212                         emit_store_dst(jd, iptr, d);
1213                         break;
1214
1215                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1216
1217                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1218                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1219                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1220                         if (s2 == d)
1221                                 M_FADD(s1, d);
1222                         else {
1223                                 M_FLTMOVE(s1, d);
1224                                 M_FADD(s2, d);
1225                         }
1226                         emit_store_dst(jd, iptr, d);
1227                         break;
1228
1229                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1230
1231                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1232                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1233                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1234                         if (s2 == d)
1235                                 M_DADD(s1, d);
1236                         else {
1237                                 M_FLTMOVE(s1, d);
1238                                 M_DADD(s2, d);
1239                         }
1240                         emit_store_dst(jd, iptr, d);
1241                         break;
1242
1243                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1244
1245                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1246                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1247                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1248                         if (s2 == d) {
1249                                 M_FLTMOVE(s2, REG_FTMP2);
1250                                 s2 = REG_FTMP2;
1251                         }
1252                         M_FLTMOVE(s1, d);
1253                         M_FSUB(s2, d);
1254                         emit_store_dst(jd, iptr, d);
1255                         break;
1256
1257                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1258
1259                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1260                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1261                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1262                         if (s2 == d) {
1263                                 M_FLTMOVE(s2, REG_FTMP2);
1264                                 s2 = REG_FTMP2;
1265                         }
1266                         M_FLTMOVE(s1, d);
1267                         M_DSUB(s2, d);
1268                         emit_store_dst(jd, iptr, d);
1269                         break;
1270
1271                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1272
1273                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1274                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1275                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1276                         if (s2 == d)
1277                                 M_FMUL(s1, d);
1278                         else {
1279                                 M_FLTMOVE(s1, d);
1280                                 M_FMUL(s2, d);
1281                         }
1282                         emit_store_dst(jd, iptr, d);
1283                         break;
1284
1285                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1286
1287                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1288                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1289                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1290                         if (s2 == d)
1291                                 M_DMUL(s1, d);
1292                         else {
1293                                 M_FLTMOVE(s1, d);
1294                                 M_DMUL(s2, d);
1295                         }
1296                         emit_store_dst(jd, iptr, d);
1297                         break;
1298
1299                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1300
1301                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1302                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1303                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1304                         if (s2 == d) {
1305                                 M_FLTMOVE(s2, REG_FTMP2);
1306                                 s2 = REG_FTMP2;
1307                         }
1308                         M_FLTMOVE(s1, d);
1309                         M_FDIV(s2, d);
1310                         emit_store_dst(jd, iptr, d);
1311                         break;
1312
1313                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1314
1315                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1316                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1317                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1318                         if (s2 == d) {
1319                                 M_FLTMOVE(s2, REG_FTMP2);
1320                                 s2 = REG_FTMP2;
1321                         }
1322                         M_FLTMOVE(s1, d);
1323                         M_DDIV(s2, d);
1324                         emit_store_dst(jd, iptr, d);
1325                         break;
1326
1327                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1328
1329                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1330                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1331                         M_CVTIF(s1, d);
1332                         emit_store_dst(jd, iptr, d);
1333                         break;
1334
1335                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1336
1337                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1338                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1339                         M_CVTID(s1, d);
1340                         emit_store_dst(jd, iptr, d);
1341                         break;
1342
1343                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1344
1345                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1346                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1347                         M_CVTLF(s1, d);
1348                         emit_store_dst(jd, iptr, d);
1349                         break;
1350
1351                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1352
1353                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1354                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1355                         M_CVTLD(s1, d);
1356                         emit_store_dst(jd, iptr, d);
1357                         break;
1358
1359                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1360
1361                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1362                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1363                         M_CVTFI(s1, d);
1364                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1365                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1366                                 ((REG_RESULT == d) ? 0 : 3);
1367                         M_BNE(disp);
1368                         M_FLTMOVE(s1, REG_FTMP1);
1369                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1370                         M_CALL(REG_ITMP2);
1371                         M_INTMOVE(REG_RESULT, d);
1372                         emit_store_dst(jd, iptr, d);
1373                         break;
1374
1375                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1376
1377                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1378                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1379                         M_CVTDI(s1, d);
1380                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1381                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1382                                 ((REG_RESULT == d) ? 0 : 3);
1383                         M_BNE(disp);
1384                         M_FLTMOVE(s1, REG_FTMP1);
1385                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1386                         M_CALL(REG_ITMP2);
1387                         M_INTMOVE(REG_RESULT, d);
1388                         emit_store_dst(jd, iptr, d);
1389                         break;
1390
1391                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1392
1393                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1394                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1395                         M_CVTFL(s1, d);
1396                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1397                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1398                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1399                                 ((REG_RESULT == d) ? 0 : 3);
1400                         M_BNE(disp);
1401                         M_FLTMOVE(s1, REG_FTMP1);
1402                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1403                         M_CALL(REG_ITMP2);
1404                         M_INTMOVE(REG_RESULT, d);
1405                         emit_store_dst(jd, iptr, d);
1406                         break;
1407
1408                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1409
1410                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1411                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1412                         M_CVTDL(s1, d);
1413                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1414                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1415                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1416                                 ((REG_RESULT == d) ? 0 : 3);
1417                         M_BNE(disp);
1418                         M_FLTMOVE(s1, REG_FTMP1);
1419                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1420                         M_CALL(REG_ITMP2);
1421                         M_INTMOVE(REG_RESULT, d);
1422                         emit_store_dst(jd, iptr, d);
1423                         break;
1424
1425                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1426
1427                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1428                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1429                         M_CVTFD(s1, d);
1430                         emit_store_dst(jd, iptr, d);
1431                         break;
1432
1433                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1434
1435                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1436                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1437                         M_CVTDF(s1, d);
1438                         emit_store_dst(jd, iptr, d);
1439                         break;
1440
1441                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1442                                           /* == => 0, < => 1, > => -1 */
1443
1444                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1445                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1446                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1447                         M_CLR(d);
1448                         M_MOV_IMM(1, REG_ITMP1);
1449                         M_MOV_IMM(-1, REG_ITMP2);
1450                         emit_ucomiss_reg_reg(cd, s1, s2);
1451                         M_CMOVULT(REG_ITMP1, d);
1452                         M_CMOVUGT(REG_ITMP2, d);
1453                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1454                         emit_store_dst(jd, iptr, d);
1455                         break;
1456
1457                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1458                                           /* == => 0, < => 1, > => -1 */
1459
1460                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1461                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1462                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1463                         M_CLR(d);
1464                         M_MOV_IMM(1, REG_ITMP1);
1465                         M_MOV_IMM(-1, REG_ITMP2);
1466                         emit_ucomiss_reg_reg(cd, s1, s2);
1467                         M_CMOVULT(REG_ITMP1, d);
1468                         M_CMOVUGT(REG_ITMP2, d);
1469                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1470                         emit_store_dst(jd, iptr, d);
1471                         break;
1472
1473                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1474                                           /* == => 0, < => 1, > => -1 */
1475
1476                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1477                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1478                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1479                         M_CLR(d);
1480                         M_MOV_IMM(1, REG_ITMP1);
1481                         M_MOV_IMM(-1, REG_ITMP2);
1482                         emit_ucomisd_reg_reg(cd, s1, s2);
1483                         M_CMOVULT(REG_ITMP1, d);
1484                         M_CMOVUGT(REG_ITMP2, d);
1485                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1486                         emit_store_dst(jd, iptr, d);
1487                         break;
1488
1489                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1490                                           /* == => 0, < => 1, > => -1 */
1491
1492                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1493                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1494                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1495                         M_CLR(d);
1496                         M_MOV_IMM(1, REG_ITMP1);
1497                         M_MOV_IMM(-1, REG_ITMP2);
1498                         emit_ucomisd_reg_reg(cd, s1, s2);
1499                         M_CMOVULT(REG_ITMP1, d);
1500                         M_CMOVUGT(REG_ITMP2, d);
1501                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1502                         emit_store_dst(jd, iptr, d);
1503                         break;
1504
1505
1506                 /* memory operations **************************************************/
1507
1508                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1509
1510                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1511                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1512                         /* implicit null-pointer check */
1513                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1514                         emit_store_dst(jd, iptr, d);
1515                         break;
1516
1517                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1518
1519                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1520                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1521                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1522                         /* implicit null-pointer check */
1523                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1524                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1525                         emit_store_dst(jd, iptr, d);
1526                         break;
1527
1528                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1529
1530                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1531                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1532                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1533                         /* implicit null-pointer check */
1534                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1535                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1536                         emit_store_dst(jd, iptr, d);
1537                         break;
1538
1539                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1540
1541                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1542                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1543                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1544                         /* implicit null-pointer check */
1545                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1546                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1547                         emit_store_dst(jd, iptr, d);
1548                         break;
1549
1550                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1551
1552                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1553                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1554                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1555                         /* implicit null-pointer check */
1556                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1557                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1558                         emit_store_dst(jd, iptr, d);
1559                         break;
1560
1561                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1562
1563                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1564                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1565                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1566                         /* implicit null-pointer check */
1567                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1568                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1569                         emit_store_dst(jd, iptr, d);
1570                         break;
1571
1572                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1573
1574                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1575                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1576                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1577                         /* implicit null-pointer check */
1578                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1579                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1580                         emit_store_dst(jd, iptr, d);
1581                         break;
1582
1583                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1584
1585                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1586                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1587                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1588                         /* implicit null-pointer check */
1589                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1590                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1591                         emit_store_dst(jd, iptr, d);
1592                         break;
1593
1594                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1595
1596                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1597                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1598                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1599                         /* implicit null-pointer check */
1600                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1601                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1602                         emit_store_dst(jd, iptr, d);
1603                         break;
1604
1605
1606                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1607
1608                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1609                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1610                         /* implicit null-pointer check */
1611                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1612                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1613                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1614                         break;
1615
1616                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1617
1618                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1619                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1620                         /* implicit null-pointer check */
1621                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1622                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1623                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1624                         break;
1625
1626                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1627
1628                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1629                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1630                         /* implicit null-pointer check */
1631                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1632                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1633                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1634                         break;
1635
1636                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1637
1638                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1639                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1640                         /* implicit null-pointer check */
1641                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1642                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1643                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
1644                         break;
1645
1646                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1647
1648                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1649                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1650                         /* implicit null-pointer check */
1651                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1652                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1653                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
1654                         break;
1655
1656                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1657
1658                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1659                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1660                         /* implicit null-pointer check */
1661                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1662                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1663                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
1664                         break;
1665
1666                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1667
1668                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1669                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1670                         /* implicit null-pointer check */
1671                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1672                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1673                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
1674                         break;
1675
1676                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1677
1678                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1679                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1680                         /* implicit null-pointer check */
1681                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1682                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1683
1684                         M_MOV(s1, REG_A0);
1685                         M_MOV(s3, REG_A1);
1686                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
1687                         M_CALL(REG_ITMP1);
1688                         emit_exception_check(cd, iptr);
1689
1690                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1691                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1692                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1693                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1694                         break;
1695
1696
1697                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1698
1699                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1700                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1701                         /* implicit null-pointer check */
1702                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1703                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1704                         break;
1705
1706                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1707
1708                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1709                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1710                         /* implicit null-pointer check */
1711                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1712                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray, data[0]), s1, s2, 1);
1713                         break;
1714
1715                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1716
1717                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1718                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1719                         /* implicit null-pointer check */
1720                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1721                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1722                         break;
1723
1724                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1725
1726                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1727                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1728                         /* implicit null-pointer check */
1729                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1730                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray, data[0]), s1, s2, 2);
1731                         break;
1732
1733                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1734
1735                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1736                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1737                         /* implicit null-pointer check */
1738                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1739
1740                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1741                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1742                         }
1743                         else {
1744                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1745                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
1746                         }
1747                         break;
1748
1749                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1750
1751                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1752                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1753                         /* implicit null-pointer check */
1754                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1755                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1756                         break;
1757
1758
1759                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1760
1761                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1762                                 uf        = iptr->sx.s23.s3.uf;
1763                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1764                                 disp      = dseg_add_unique_address(cd, NULL);
1765                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1766
1767                                 /* must be calculated before codegen_add_patch_ref */
1768
1769                                 if (opt_shownops)
1770                                         disp -= PATCHER_CALL_SIZE;
1771
1772 /*                              PROFILE_CYCLE_STOP; */
1773
1774                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1775
1776 /*                              PROFILE_CYCLE_START; */
1777                         }
1778                         else {
1779                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1780                                 fieldtype = fi->type;
1781                                 disp      = dseg_add_address(cd, &(fi->value));
1782                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1783
1784                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1785                                         PROFILE_CYCLE_STOP;
1786
1787                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1788
1789                                         PROFILE_CYCLE_START;
1790
1791                                         if (opt_shownops)
1792                                                 disp -= PATCHER_CALL_SIZE;
1793                                 }
1794                         }
1795
1796                         /* This approach is much faster than moving the field
1797                            address inline into a register. */
1798
1799                         M_ALD(REG_ITMP1, RIP, disp);
1800
1801                         switch (fieldtype) {
1802                         case TYPE_INT:
1803                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1804                                 M_ILD(d, REG_ITMP1, 0);
1805                                 break;
1806                         case TYPE_LNG:
1807                         case TYPE_ADR:
1808                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1809                                 M_LLD(d, REG_ITMP1, 0);
1810                                 break;
1811                         case TYPE_FLT:
1812                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1813                                 M_FLD(d, REG_ITMP1, 0);
1814                                 break;
1815                         case TYPE_DBL:
1816                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1817                                 M_DLD(d, REG_ITMP1, 0);
1818                                 break;
1819                         }
1820                         emit_store_dst(jd, iptr, d);
1821                         break;
1822
1823                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1824
1825                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1826                                 uf        = iptr->sx.s23.s3.uf;
1827                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1828                                 disp      = dseg_add_unique_address(cd, NULL);
1829                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1830
1831                                 /* must be calculated before codegen_add_patch_ref */
1832
1833                                 if (opt_shownops)
1834                                         disp -= PATCHER_CALL_SIZE;
1835
1836 /*                              PROFILE_CYCLE_STOP; */
1837
1838                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1839
1840 /*                              PROFILE_CYCLE_START; */
1841                         }
1842                         else {
1843                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1844                                 fieldtype = fi->type;
1845                                 disp      = dseg_add_address(cd, &(fi->value));
1846                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1847
1848                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1849                                         PROFILE_CYCLE_STOP;
1850
1851                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1852
1853                                         PROFILE_CYCLE_START;
1854
1855                                         if (opt_shownops)
1856                                                 disp -= PATCHER_CALL_SIZE;
1857                                 }
1858                         }
1859
1860                         /* This approach is much faster than moving the field
1861                            address inline into a register. */
1862
1863                         M_ALD(REG_ITMP1, RIP, disp);
1864
1865                         switch (fieldtype) {
1866                         case TYPE_INT:
1867                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1868                                 M_IST(s1, REG_ITMP1, 0);
1869                                 break;
1870                         case TYPE_LNG:
1871                         case TYPE_ADR:
1872                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1873                                 M_LST(s1, REG_ITMP1, 0);
1874                                 break;
1875                         case TYPE_FLT:
1876                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1877                                 M_FST(s1, REG_ITMP1, 0);
1878                                 break;
1879                         case TYPE_DBL:
1880                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1881                                 M_DST(s1, REG_ITMP1, 0);
1882                                 break;
1883                         }
1884                         break;
1885
1886                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1887                                           /* val = value (in current instruction)     */
1888                                           /* following NOP)                           */
1889
1890                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1891                                 uf        = iptr->sx.s23.s3.uf;
1892                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1893                                 disp      = dseg_add_unique_address(cd, NULL);
1894                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1895
1896                                 /* must be calculated before codegen_add_patch_ref */
1897
1898                                 if (opt_shownops)
1899                                         disp -= PATCHER_CALL_SIZE;
1900
1901 /*                              PROFILE_CYCLE_STOP; */
1902
1903                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1904
1905 /*                              PROFILE_CYCLE_START; */
1906                         }
1907                         else {
1908                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1909                                 fieldtype = fi->type;
1910                                 disp      = dseg_add_address(cd, &(fi->value));
1911                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1912
1913                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1914                                         PROFILE_CYCLE_STOP;
1915
1916                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1917
1918                                         PROFILE_CYCLE_START;
1919
1920                                         if (opt_shownops)
1921                                                 disp -= PATCHER_CALL_SIZE;
1922                                 }
1923                         }
1924
1925                         /* This approach is much faster than moving the field
1926                            address inline into a register. */
1927
1928                         M_ALD(REG_ITMP1, RIP, disp);
1929
1930                         switch (fieldtype) {
1931                         case TYPE_INT:
1932                         case TYPE_FLT:
1933                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1934                                 break;
1935                         case TYPE_LNG:
1936                         case TYPE_ADR:
1937                         case TYPE_DBL:
1938                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1939                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1940                                 else {
1941                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1942                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1943                                 }
1944                                 break;
1945                         }
1946                         break;
1947
1948                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1949
1950                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1951
1952                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1953                                 uf        = iptr->sx.s23.s3.uf;
1954                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1955                                 disp      = 0;
1956
1957 /*                              PROFILE_CYCLE_STOP; */
1958
1959                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1960
1961 /*                              PROFILE_CYCLE_START; */
1962                         }
1963                         else {
1964                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1965                                 fieldtype = fi->type;
1966                                 disp      = fi->offset;
1967                         }
1968
1969                         /* implicit null-pointer check */
1970                         switch (fieldtype) {
1971                         case TYPE_INT:
1972                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1973                                 M_ILD32(d, s1, disp);
1974                                 break;
1975                         case TYPE_LNG:
1976                         case TYPE_ADR:
1977                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1978                                 M_LLD32(d, s1, disp);
1979                                 break;
1980                         case TYPE_FLT:
1981                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1982                                 M_FLD32(d, s1, disp);
1983                                 break;
1984                         case TYPE_DBL:
1985                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1986                                 M_DLD32(d, s1, disp);
1987                                 break;
1988                         }
1989                         emit_store_dst(jd, iptr, d);
1990                         break;
1991
1992                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1993
1994                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1995                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1996
1997                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1998                                 uf        = iptr->sx.s23.s3.uf;
1999                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2000                                 disp      = 0;
2001
2002 /*                              PROFILE_CYCLE_STOP; */
2003
2004                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
2005
2006 /*                              PROFILE_CYCLE_START; */
2007                         }
2008                         else {
2009                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2010                                 fieldtype = fi->type;
2011                                 disp      = fi->offset;
2012                         }
2013
2014                         /* implicit null-pointer check */
2015                         switch (fieldtype) {
2016                         case TYPE_INT:
2017                                 M_IST32(s2, s1, disp);
2018                                 break;
2019                         case TYPE_LNG:
2020                         case TYPE_ADR:
2021                                 M_LST32(s2, s1, disp);
2022                                 break;
2023                         case TYPE_FLT:
2024                                 M_FST32(s2, s1, disp);
2025                                 break;
2026                         case TYPE_DBL:
2027                                 M_DST32(s2, s1, disp);
2028                                 break;
2029                         }
2030                         break;
2031
2032                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2033                                           /* val = value (in current instruction)     */
2034                                           /* following NOP)                           */
2035
2036                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2037
2038                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2039                                 uf        = iptr->sx.s23.s3.uf;
2040                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2041                                 disp      = 0;
2042
2043 /*                              PROFILE_CYCLE_STOP; */
2044
2045                                 codegen_add_patch_ref(cd, PATCHER_putfieldconst, uf, 0);
2046
2047 /*                              PROFILE_CYCLE_START; */
2048                         }
2049                         else {
2050                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2051                                 fieldtype = fi->type;
2052                                 disp      = fi->offset;
2053                         }
2054
2055                         /* implicit null-pointer check */
2056                         switch (fieldtype) {
2057                         case TYPE_INT:
2058                         case TYPE_FLT:
2059                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2060                                 break;
2061                         case TYPE_LNG:
2062                         case TYPE_ADR:
2063                         case TYPE_DBL:
2064                                 /* XXX why no check for IS_IMM32? */
2065                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2066                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2067                                 break;
2068                         }
2069                         break;
2070
2071
2072                 /* branch operations **************************************************/
2073
2074                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2075
2076                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2077                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2078
2079                         PROFILE_CYCLE_STOP;
2080
2081 #ifdef ENABLE_VERIFIER
2082                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2083                                 uc = iptr->sx.s23.s2.uc;
2084
2085                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2086                         }
2087 #endif /* ENABLE_VERIFIER */
2088
2089                         M_CALL_IMM(0);                            /* passing exception pc */
2090                         M_POP(REG_ITMP2_XPC);
2091
2092                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2093                         M_JMP(REG_ITMP3);
2094                         break;
2095
2096                 case ICMD_GOTO:         /* ... ==> ...                                */
2097                 case ICMD_RET:
2098
2099                         emit_br(cd, iptr->dst.block);
2100                         ALIGNCODENOP;
2101                         break;
2102
2103                 case ICMD_JSR:          /* ... ==> ...                                */
2104
2105                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2106                         ALIGNCODENOP;
2107                         break;
2108
2109                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2110                 case ICMD_IFNONNULL:
2111
2112                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2113                         M_TEST(s1);
2114                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2115                         break;
2116
2117                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2118                 case ICMD_IFLT:
2119                 case ICMD_IFLE:
2120                 case ICMD_IFNE:
2121                 case ICMD_IFGT:
2122                 case ICMD_IFGE:
2123
2124                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2125                         M_ICMP_IMM(iptr->sx.val.i, s1);
2126                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2127                         break;
2128
2129                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2130                 case ICMD_IF_LNE:
2131                 case ICMD_IF_LLT:
2132                 case ICMD_IF_LGE:
2133                 case ICMD_IF_LGT:
2134                 case ICMD_IF_LLE:
2135
2136                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2137                         if (IS_IMM32(iptr->sx.val.l))
2138                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2139                         else {
2140                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2141                                 M_LCMP(REG_ITMP2, s1);
2142                         }
2143                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2144                         break;
2145
2146                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2147                 case ICMD_IF_ICMPNE:
2148                 case ICMD_IF_ICMPLT:
2149                 case ICMD_IF_ICMPGE:
2150                 case ICMD_IF_ICMPGT:
2151                 case ICMD_IF_ICMPLE:
2152
2153                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2154                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2155                         M_ICMP(s2, s1);
2156                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2157                         break;
2158
2159                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2160                 case ICMD_IF_ACMPNE:
2161
2162                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2163                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2164                         M_LCMP(s2, s1);
2165                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2166                         break;
2167
2168                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2169                 case ICMD_IF_LCMPNE:
2170                 case ICMD_IF_LCMPLT:
2171                 case ICMD_IF_LCMPGE:
2172                 case ICMD_IF_LCMPGT:
2173                 case ICMD_IF_LCMPLE:
2174
2175                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2176                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2177                         M_LCMP(s2, s1);
2178                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2179                         break;
2180
2181                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2182                 case ICMD_LRETURN:
2183
2184                         REPLACEMENT_POINT_RETURN(cd, iptr);
2185                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2186                         M_INTMOVE(s1, REG_RESULT);
2187                         goto nowperformreturn;
2188
2189                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2190
2191                         REPLACEMENT_POINT_RETURN(cd, iptr);
2192                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2193                         M_INTMOVE(s1, REG_RESULT);
2194
2195 #ifdef ENABLE_VERIFIER
2196                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2197                                 uc = iptr->sx.s23.s2.uc;
2198
2199                                 PROFILE_CYCLE_STOP;
2200
2201                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2202
2203                                 PROFILE_CYCLE_START;
2204                         }
2205 #endif /* ENABLE_VERIFIER */
2206                         goto nowperformreturn;
2207
2208                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2209                 case ICMD_DRETURN:
2210
2211                         REPLACEMENT_POINT_RETURN(cd, iptr);
2212                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2213                         M_FLTMOVE(s1, REG_FRESULT);
2214                         goto nowperformreturn;
2215
2216                 case ICMD_RETURN:      /* ...  ==> ...                                */
2217
2218                         REPLACEMENT_POINT_RETURN(cd, iptr);
2219
2220 nowperformreturn:
2221                         {
2222                         s4 i, p;
2223
2224                         p = cd->stackframesize;
2225
2226 #if !defined(NDEBUG)
2227                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2228                                 emit_verbosecall_exit(jd);
2229 #endif /* !defined(NDEBUG) */
2230
2231 #if defined(ENABLE_THREADS)
2232                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2233                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2234
2235                                 /* we need to save the proper return value */
2236                                 switch (iptr->opc) {
2237                                 case ICMD_IRETURN:
2238                                 case ICMD_ARETURN:
2239                                 case ICMD_LRETURN:
2240                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2241                                         break;
2242                                 case ICMD_FRETURN:
2243                                 case ICMD_DRETURN:
2244                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2245                                         break;
2246                                 }
2247
2248                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2249                                 M_CALL(REG_ITMP1);
2250
2251                                 /* and now restore the proper return value */
2252                                 switch (iptr->opc) {
2253                                 case ICMD_IRETURN:
2254                                 case ICMD_ARETURN:
2255                                 case ICMD_LRETURN:
2256                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2257                                         break;
2258                                 case ICMD_FRETURN:
2259                                 case ICMD_DRETURN:
2260                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2261                                         break;
2262                                 }
2263                         }
2264 #endif
2265
2266                         /* restore saved registers */
2267
2268                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2269                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2270                         }
2271                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2272                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2273                         }
2274
2275                         /* deallocate stack */
2276
2277                         if (cd->stackframesize)
2278                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2279
2280                         /* generate method profiling code */
2281
2282                         PROFILE_CYCLE_STOP;
2283
2284                         M_RET;
2285                         }
2286                         break;
2287
2288
2289                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2290                         {
2291                                 s4 i, l;
2292                                 branch_target_t *table;
2293
2294                                 table = iptr->dst.table;
2295
2296                                 l = iptr->sx.s23.s2.tablelow;
2297                                 i = iptr->sx.s23.s3.tablehigh;
2298
2299                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2300                                 M_INTMOVE(s1, REG_ITMP1);
2301
2302                                 if (l != 0)
2303                                         M_ISUB_IMM(l, REG_ITMP1);
2304
2305                                 /* number of targets */
2306                                 i = i - l + 1;
2307
2308                 /* range check */
2309
2310                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2311                                 emit_bugt(cd, table[0].block);
2312
2313                                 /* build jump table top down and use address of lowest entry */
2314
2315                                 table += i;
2316
2317                                 while (--i >= 0) {
2318                                         dseg_add_target(cd, table->block);
2319                                         --table;
2320                                 }
2321
2322                                 /* length of dataseg after last dseg_add_target is used
2323                                    by load */
2324
2325                                 M_MOV_IMM(0, REG_ITMP2);
2326                                 dseg_adddata(cd);
2327                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2328                                 M_JMP(REG_ITMP1);
2329                         }
2330                         break;
2331
2332
2333                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2334                         {
2335                                 s4 i;
2336                                 lookup_target_t *lookup;
2337
2338                                 lookup = iptr->dst.lookup;
2339
2340                                 i = iptr->sx.s23.s2.lookupcount;
2341
2342                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2343                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2344
2345                                 while (--i >= 0) {
2346                                         M_ICMP_IMM(lookup->value, s1);
2347                                         emit_beq(cd, lookup->target.block);
2348                                         lookup++;
2349                                 }
2350
2351                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2352                                 ALIGNCODENOP;
2353                         }
2354                         break;
2355
2356
2357                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2358
2359                         bte = iptr->sx.s23.s3.bte;
2360                         md  = bte->md;
2361                         goto gen_method;
2362
2363                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2364
2365                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2366                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2367                 case ICMD_INVOKEINTERFACE:
2368
2369                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2370
2371                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2372                                 lm = NULL;
2373                                 um = iptr->sx.s23.s3.um;
2374                                 md = um->methodref->parseddesc.md;
2375                         }
2376                         else {
2377                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2378                                 um = NULL;
2379                                 md = lm->parseddesc;
2380                         }
2381
2382 gen_method:
2383                         s3 = md->paramcount;
2384
2385                         MCODECHECK((20 * s3) + 128);
2386
2387                         /* copy arguments to registers or stack location */
2388
2389                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2390                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2391
2392                                 /* Already Preallocated (ARGVAR) ? */
2393                                 if (var->flags & PREALLOC)
2394                                         continue;
2395
2396                                 if (IS_INT_LNG_TYPE(var->type)) {
2397                                         if (!md->params[s3].inmemory) {
2398                                                 s1 = rd->argintregs[md->params[s3].regoff];
2399                                                 d = emit_load(jd, iptr, var, s1);
2400                                                 M_INTMOVE(d, s1);
2401                                         }
2402                                         else {
2403                                                 d = emit_load(jd, iptr, var, REG_ITMP1);
2404                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
2405                                         }
2406                                 }
2407                                 else {
2408                                         if (!md->params[s3].inmemory) {
2409                                                 s1 = rd->argfltregs[md->params[s3].regoff];
2410                                                 d = emit_load(jd, iptr, var, s1);
2411                                                 M_FLTMOVE(d, s1);
2412                                         }
2413                                         else {
2414                                                 d = emit_load(jd, iptr, var, REG_FTMP1);
2415
2416                                                 if (IS_2_WORD_TYPE(var->type))
2417                                                         M_DST(d, REG_SP, md->params[s3].regoff * 8);
2418                                                 else
2419                                                         M_FST(d, REG_SP, md->params[s3].regoff * 8);
2420                                         }
2421                                 }
2422                         }
2423
2424                         /* generate method profiling code */
2425
2426                         PROFILE_CYCLE_STOP;
2427
2428                         switch (iptr->opc) {
2429                         case ICMD_BUILTIN:
2430                                 M_MOV_IMM(bte->fp, REG_ITMP1);
2431                                 M_CALL(REG_ITMP1);
2432
2433                                 emit_exception_check(cd, iptr);
2434                                 break;
2435
2436                         case ICMD_INVOKESPECIAL:
2437                                 emit_nullpointer_check(cd, iptr, REG_A0);
2438                                 /* fall through */
2439
2440                         case ICMD_INVOKESTATIC:
2441                                 if (lm == NULL) {
2442                                         disp = dseg_add_unique_address(cd, NULL);
2443                                         disp = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
2444
2445                                         /* must be calculated before codegen_add_patch_ref */
2446
2447                                         if (opt_shownops)
2448                                                 disp -= PATCHER_CALL_SIZE;
2449
2450                                         codegen_add_patch_ref(cd, PATCHER_invokestatic_special,
2451                                                                                   um, disp);
2452
2453 /*                                      a = 0; */
2454                                 }
2455                                 else {
2456                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2457                                         disp = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
2458
2459 /*                                      a = (ptrint) lm->stubroutine; */
2460                                 }
2461
2462 /*                              M_MOV_IMM(a, REG_ITMP2); */
2463                                 M_ALD(REG_ITMP2, RIP, disp);
2464                                 M_CALL(REG_ITMP2);
2465                                 break;
2466
2467                         case ICMD_INVOKEVIRTUAL:
2468                                 if (lm == NULL) {
2469                                         codegen_add_patch_ref(cd, PATCHER_invokevirtual, um, 0);
2470
2471                                         s1 = 0;
2472                                 }
2473                                 else {
2474                                         s1 = OFFSET(vftbl_t, table[0]) +
2475                                                 sizeof(methodptr) * lm->vftblindex;
2476                                 }
2477
2478                                 /* implicit null-pointer check */
2479                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_objectheader, vftbl));
2480                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2481                                 M_CALL(REG_ITMP3);
2482                                 break;
2483
2484                         case ICMD_INVOKEINTERFACE:
2485                                 if (lm == NULL) {
2486                                         codegen_add_patch_ref(cd, PATCHER_invokeinterface, um, 0);
2487
2488                                         s1 = 0;
2489                                         s2 = 0;
2490                                 }
2491                                 else {
2492                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2493                                                 sizeof(methodptr) * lm->class->index;
2494
2495                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2496                                 }
2497
2498                                 /* implicit null-pointer check */
2499                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_objectheader, vftbl));
2500                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2501                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2502                                 M_CALL(REG_ITMP3);
2503                                 break;
2504                         }
2505
2506                         /* generate method profiling code */
2507
2508                         PROFILE_CYCLE_START;
2509
2510                         /* store size of call code in replacement point */
2511
2512                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2513
2514                         /* store return value */
2515
2516                         switch (md->returntype.type) {
2517                         case TYPE_INT:
2518                         case TYPE_LNG:
2519                         case TYPE_ADR:
2520                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2521                                 M_INTMOVE(REG_RESULT, s1);
2522                                 emit_store_dst(jd, iptr, s1);
2523                                 break;
2524                         case TYPE_FLT:
2525                         case TYPE_DBL:
2526                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2527                                 M_FLTMOVE(REG_FRESULT, s1);
2528                                 emit_store_dst(jd, iptr, s1);
2529                                 break;
2530                         default:
2531                                 /* TYPE_VOID */
2532                                 break;
2533                         }
2534                         break;
2535
2536
2537                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2538
2539                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2540                                 /* object type cast-check */
2541
2542                                 classinfo *super;
2543                                 vftbl_t   *supervftbl;
2544                                 s4         superindex;
2545
2546                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2547                                         super      = NULL;
2548                                         superindex = 0;
2549                                         supervftbl = NULL;
2550                                 }
2551                                 else {
2552                                         super      = iptr->sx.s23.s3.c.cls;
2553                                         superindex = super->index;
2554                                         supervftbl = super->vftbl;
2555                                 }
2556
2557 #if defined(ENABLE_THREADS)
2558                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
2559 #endif
2560                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2561
2562                                 /* if class is not resolved, check which code to call */
2563
2564                                 if (super == NULL) {
2565                                         M_TEST(s1);
2566                                         emit_label_beq(cd, BRANCH_LABEL_1);
2567
2568                                         codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2569                                                                                   iptr->sx.s23.s3.c.ref, 0);
2570
2571                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2572                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2573                                         emit_label_beq(cd, BRANCH_LABEL_2);
2574                                 }
2575
2576                                 /* interface checkcast code */
2577
2578                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2579                                         if (super != NULL) {
2580                                                 M_TEST(s1);
2581                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2582                                         }
2583
2584                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2585
2586                                         if (super == NULL) {
2587                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_interface,
2588                                                                                           iptr->sx.s23.s3.c.ref,
2589                                                                                           0);
2590                                         }
2591
2592                                         M_ILD32(REG_ITMP3,
2593                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2594                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2595                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2596
2597                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2598                                                         OFFSET(vftbl_t, interfacetable[0]) -
2599                                                         superindex * sizeof(methodptr*));
2600                                         M_TEST(REG_ITMP3);
2601                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2602
2603                                         if (super == NULL)
2604                                                 emit_label_br(cd, BRANCH_LABEL_4);
2605                                         else
2606                                                 emit_label(cd, BRANCH_LABEL_3);
2607                                 }
2608
2609                                 /* class checkcast code */
2610
2611                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2612                                         if (super == NULL) {
2613                                                 emit_label(cd, BRANCH_LABEL_2);
2614                                         }
2615                                         else {
2616                                                 M_TEST(s1);
2617                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2618                                         }
2619
2620                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2621
2622                                         if (super == NULL) {
2623                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_class,
2624                                                                                           iptr->sx.s23.s3.c.ref,
2625                                                                                           0);
2626                                         }
2627
2628                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2629 #if defined(ENABLE_THREADS)
2630                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
2631 #endif
2632                                         M_ILD32(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2633
2634                                         /*                                      if (s1 != REG_ITMP1) { */
2635                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2636                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2637                                         /*                                                                                              REG_ITMP1); */
2638                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2639                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2640                                         /*                                                                                              REG_ITMP3); */
2641                                         /*  #if defined(ENABLE_THREADS) */
2642                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2643                                         /*  #endif */
2644                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2645
2646                                         /*                                      } else { */
2647
2648                                         M_ILD32(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
2649                                         M_ISUB(REG_ITMP3, REG_ITMP2);
2650                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2651                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2652                                         /*                                      } */
2653 #if defined(ENABLE_THREADS)
2654                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
2655 #endif
2656                                         M_ICMP(REG_ITMP3, REG_ITMP2);
2657                                         emit_classcast_check(cd, iptr, BRANCH_UGT, REG_ITMP3, s1);
2658
2659                                         if (super != NULL)
2660                                                 emit_label(cd, BRANCH_LABEL_5);
2661                                 }
2662
2663                                 if (super == NULL) {
2664                                         emit_label(cd, BRANCH_LABEL_1);
2665                                         emit_label(cd, BRANCH_LABEL_4);
2666                                 }
2667
2668                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2669                         }
2670                         else {
2671                                 /* array type cast-check */
2672
2673                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2674                                 M_INTMOVE(s1, REG_A0);
2675
2676                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2677                                         codegen_add_patch_ref(cd, PATCHER_builtin_arraycheckcast,
2678                                                                                   iptr->sx.s23.s3.c.ref, 0);
2679                                 }
2680
2681                                 M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2682                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2683                                 M_CALL(REG_ITMP1);
2684
2685                                 /* s1 may have been destroyed over the function call */
2686                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2687                                 M_TEST(REG_RESULT);
2688                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2689
2690                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2691                         }
2692
2693                         M_INTMOVE(s1, d);
2694                         emit_store_dst(jd, iptr, d);
2695                         break;
2696
2697                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2698
2699                         {
2700                         classinfo *super;
2701                         vftbl_t   *supervftbl;
2702                         s4         superindex;
2703
2704                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2705                                 super      = NULL;
2706                                 superindex = 0;
2707                                 supervftbl = NULL;
2708
2709                         } else {
2710                                 super      = iptr->sx.s23.s3.c.cls;
2711                                 superindex = super->index;
2712                                 supervftbl = super->vftbl;
2713                         }
2714
2715 #if defined(ENABLE_THREADS)
2716             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
2717 #endif
2718
2719                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2720                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2721
2722                         if (s1 == d) {
2723                                 M_INTMOVE(s1, REG_ITMP1);
2724                                 s1 = REG_ITMP1;
2725                         }
2726
2727                         M_CLR(d);
2728
2729                         /* if class is not resolved, check which code to call */
2730
2731                         if (super == NULL) {
2732                                 M_TEST(s1);
2733                                 emit_label_beq(cd, BRANCH_LABEL_1);
2734
2735                                 codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2736                                                                           iptr->sx.s23.s3.c.ref, 0);
2737
2738                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2739                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2740                                 emit_label_beq(cd, BRANCH_LABEL_2);
2741                         }
2742
2743                         /* interface instanceof code */
2744
2745                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2746                                 if (super != NULL) {
2747                                         M_TEST(s1);
2748                                         emit_label_beq(cd, BRANCH_LABEL_3);
2749                                 }
2750
2751                                 M_ALD(REG_ITMP1, s1, OFFSET(java_objectheader, vftbl));
2752
2753                                 if (super == NULL) {
2754                                         codegen_add_patch_ref(cd, PATCHER_instanceof_interface,
2755                                                                                   iptr->sx.s23.s3.c.ref, 0);
2756                                 }
2757
2758                                 M_ILD32(REG_ITMP3,
2759                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2760                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2761
2762                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2763
2764                                 M_BLE(a);
2765                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2766                                                 OFFSET(vftbl_t, interfacetable[0]) -
2767                                                 superindex * sizeof(methodptr*));
2768                                 M_TEST(REG_ITMP1);
2769                                 M_SETNE(d);
2770
2771                                 if (super == NULL)
2772                                         emit_label_br(cd, BRANCH_LABEL_4);
2773                                 else
2774                                         emit_label(cd, BRANCH_LABEL_3);
2775                         }
2776
2777                         /* class instanceof code */
2778
2779                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2780                                 if (super == NULL) {
2781                                         emit_label(cd, BRANCH_LABEL_2);
2782                                 }
2783                                 else {
2784                                         M_TEST(s1);
2785                                         emit_label_beq(cd, BRANCH_LABEL_5);
2786                                 }
2787
2788                                 M_ALD(REG_ITMP1, s1, OFFSET(java_objectheader, vftbl));
2789
2790                                 if (super == NULL) {
2791                                         codegen_add_patch_ref(cd, PATCHER_instanceof_class,
2792                                                                                   iptr->sx.s23.s3.c.ref, 0);
2793                                 }
2794
2795                                 M_MOV_IMM(supervftbl, REG_ITMP2);
2796
2797 #if defined(ENABLE_THREADS)
2798                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
2799 #endif
2800
2801                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
2802                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2803                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2804
2805 #if defined(ENABLE_THREADS)
2806                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
2807 #endif
2808
2809                                 M_ISUB(REG_ITMP2, REG_ITMP1);
2810                                 M_CLR(d); /* may be REG_ITMP2 */
2811                                 M_ICMP(REG_ITMP3, REG_ITMP1);
2812                                 M_SETULE(d);
2813
2814                                 if (super != NULL)
2815                                         emit_label(cd, BRANCH_LABEL_5);
2816                         }
2817
2818                         if (super == NULL) {
2819                                 emit_label(cd, BRANCH_LABEL_1);
2820                                 emit_label(cd, BRANCH_LABEL_4);
2821                         }
2822
2823                         emit_store_dst(jd, iptr, d);
2824                         }
2825                         break;
2826
2827                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2828
2829                         /* check for negative sizes and copy sizes to stack if necessary  */
2830
2831                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2832
2833                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2834
2835                                 /* copy SAVEDVAR sizes to stack */
2836                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2837
2838                                 /* Already Preallocated? */
2839                                 if (!(var->flags & PREALLOC)) {
2840                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2841                                         M_LST(s2, REG_SP, s1 * 8);
2842                                 }
2843                         }
2844
2845                         /* is a patcher function set? */
2846
2847                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2848                                 codegen_add_patch_ref(cd, PATCHER_builtin_multianewarray,
2849                                                                           iptr->sx.s23.s3.c.ref, 0);
2850                         }
2851
2852                         /* a0 = dimension count */
2853
2854                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2855
2856                         /* a1 = classinfo */
2857
2858                         M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2859
2860                         /* a2 = pointer to dimensions = stack pointer */
2861
2862                         M_MOV(REG_SP, REG_A2);
2863
2864                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2865                         M_CALL(REG_ITMP1);
2866
2867                         /* check for exception before result assignment */
2868
2869                         emit_exception_check(cd, iptr);
2870
2871                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2872                         M_INTMOVE(REG_RESULT, s1);
2873                         emit_store_dst(jd, iptr, s1);
2874                         break;
2875
2876                 default:
2877                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2878                                                                                    iptr->opc);
2879                         return false;
2880         } /* switch */
2881
2882         } /* for instruction */
2883
2884         MCODECHECK(512); /* XXX require a lower number? */
2885
2886         /* At the end of a basic block we may have to append some nops,
2887            because the patcher stub calling code might be longer than the
2888            actual instruction. So codepatching does not change the
2889            following block unintentionally. */
2890
2891         if (cd->mcodeptr < cd->lastmcodeptr) {
2892                 while (cd->mcodeptr < cd->lastmcodeptr) {
2893                         M_NOP;
2894                 }
2895         }
2896
2897         } /* if (bptr -> flags >= BBREACHED) */
2898         } /* for basic block */
2899
2900         dseg_createlinenumbertable(cd);
2901
2902         /* generate stubs */
2903
2904         emit_patcher_stubs(jd);
2905         REPLACEMENT_EMIT_STUBS(jd);
2906
2907         /* everything's ok */
2908
2909         return true;
2910 }
2911
2912
2913 /* createcompilerstub **********************************************************
2914
2915    Creates a stub routine which calls the compiler.
2916
2917 *******************************************************************************/
2918
2919 #define COMPILERSTUB_DATASIZE    3 * SIZEOF_VOID_P
2920 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
2921
2922 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
2923
2924
2925 u1 *createcompilerstub(methodinfo *m)
2926 {
2927         u1          *s;                     /* memory to hold the stub            */
2928         ptrint      *d;
2929         codegendata *cd;
2930         s4           dumpsize;
2931
2932         s = CNEW(u1, COMPILERSTUB_SIZE);
2933
2934         /* set data pointer and code pointer */
2935
2936         d = (ptrint *) s;
2937         s = s + COMPILERSTUB_DATASIZE;
2938
2939         /* mark start of dump memory area */
2940
2941         dumpsize = dump_size();
2942
2943         cd = DNEW(codegendata);
2944         cd->mcodeptr = s;
2945
2946         /* The codeinfo pointer is actually a pointer to the
2947            methodinfo. This fakes a codeinfo structure. */
2948
2949         d[0] = (ptrint) asm_call_jit_compiler;
2950         d[1] = (ptrint) m;
2951         d[2] = (ptrint) &d[1];                                    /* fake code->m */
2952
2953         /* code for the stub */
2954
2955         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
2956         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
2957         M_JMP(REG_ITMP3);
2958
2959 #if defined(ENABLE_STATISTICS)
2960         if (opt_stat)
2961                 count_cstub_len += COMPILERSTUB_SIZE;
2962 #endif
2963
2964         /* release dump area */
2965
2966         dump_release(dumpsize);
2967
2968         return s;
2969 }
2970
2971
2972 /* createnativestub ************************************************************
2973
2974    Creates a stub routine which calls a native method.
2975
2976 *******************************************************************************/
2977
2978 u1 *createnativestub(functionptr f, jitdata *jd, methoddesc *nmd)
2979 {
2980         methodinfo   *m;
2981         codeinfo     *code;
2982         codegendata  *cd;
2983         registerdata *rd;
2984         methoddesc   *md;
2985         s4            nativeparams;
2986         s4            i, j;                 /* count variables                    */
2987         s4            t;
2988         s4            s1, s2;
2989
2990         /* get required compiler data */
2991
2992         m    = jd->m;
2993         code = jd->code;
2994         cd   = jd->cd;
2995         rd   = jd->rd;
2996
2997         /* initialize variables */
2998
2999         md = m->parseddesc;
3000         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
3001
3002         /* calculate stack frame size */
3003
3004         cd->stackframesize =
3005                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
3006                 sizeof(localref_table) / SIZEOF_VOID_P +
3007                 INT_ARG_CNT + FLT_ARG_CNT +
3008                 1 +                       /* functionptr, TODO: store in data segment */
3009                 nmd->memuse;
3010
3011         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
3012
3013         /* create method header */
3014
3015         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
3016         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
3017         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
3018         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
3019         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
3020         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
3021         (void) dseg_addlinenumbertablesize(cd);
3022         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
3023
3024 #if defined(ENABLE_PROFILING)
3025         /* generate native method profiling code */
3026
3027         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3028                 /* count frequency */
3029
3030                 M_MOV_IMM(code, REG_ITMP3);
3031                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3032         }
3033 #endif
3034
3035         /* generate stub code */
3036
3037         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3038
3039 #if !defined(NDEBUG)
3040         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3041                 emit_verbosecall_enter(jd);
3042 #endif
3043
3044         /* get function address (this must happen before the stackframeinfo) */
3045
3046 #if !defined(WITH_STATIC_CLASSPATH)
3047         if (f == NULL)
3048                 codegen_add_patch_ref(cd, PATCHER_resolve_native, m, 0);
3049 #endif
3050
3051         M_MOV_IMM(f, REG_ITMP3);
3052
3053
3054         /* save integer and float argument registers */
3055
3056         for (i = 0, j = 0; i < md->paramcount; i++) {
3057                 if (!md->params[i].inmemory) {
3058                         s1 = md->params[i].regoff;
3059
3060                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3061                                 M_LST(rd->argintregs[s1], REG_SP, j * 8);
3062                         else
3063                                 M_DST(rd->argfltregs[s1], REG_SP, j * 8);
3064
3065                         j++;
3066                 }
3067         }
3068
3069         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3070
3071         /* create dynamic stack info */
3072
3073         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3074         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3075         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3076         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3077         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3078         M_CALL(REG_ITMP1);
3079
3080         /* restore integer and float argument registers */
3081
3082         for (i = 0, j = 0; i < md->paramcount; i++) {
3083                 if (!md->params[i].inmemory) {
3084                         s1 = md->params[i].regoff;
3085
3086                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3087                                 M_LLD(rd->argintregs[s1], REG_SP, j * 8);
3088                         else
3089                                 M_DLD(rd->argfltregs[s1], REG_SP, j * 8);
3090
3091                         j++;
3092                 }
3093         }
3094
3095         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3096
3097
3098         /* copy or spill arguments to new locations */
3099
3100         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3101                 t = md->paramtypes[i].type;
3102
3103                 if (IS_INT_LNG_TYPE(t)) {
3104                         if (!md->params[i].inmemory) {
3105                                 s1 = rd->argintregs[md->params[i].regoff];
3106
3107                                 if (!nmd->params[j].inmemory) {
3108                                         s2 = rd->argintregs[nmd->params[j].regoff];
3109                                         M_INTMOVE(s1, s2);
3110
3111                                 } else {
3112                                         s2 = nmd->params[j].regoff;
3113                                         M_LST(s1, REG_SP, s2 * 8);
3114                                 }
3115
3116                         } else {
3117                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3118                                 s2 = nmd->params[j].regoff;
3119                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
3120                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
3121                         }
3122
3123                 } else {
3124                         /* We only copy spilled float arguments, as the float argument    */
3125                         /* registers keep unchanged.                                      */
3126
3127                         if (md->params[i].inmemory) {
3128                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3129                                 s2 = nmd->params[j].regoff;
3130
3131                                 if (IS_2_WORD_TYPE(t)) {
3132                                         M_DLD(REG_FTMP1, REG_SP, s1 * 8);
3133                                         M_DST(REG_FTMP1, REG_SP, s2 * 8);
3134                                 } else {
3135                                         M_FLD(REG_FTMP1, REG_SP, s1 * 8);
3136                                         M_FST(REG_FTMP1, REG_SP, s2 * 8);
3137                                 }
3138                         }
3139                 }
3140         }
3141
3142         /* put class into second argument register */
3143
3144         if (m->flags & ACC_STATIC)
3145                 M_MOV_IMM(m->class, REG_A1);
3146
3147         /* put env into first argument register */
3148
3149         M_MOV_IMM(_Jv_env, REG_A0);
3150
3151         /* do the native function call */
3152
3153         M_CALL(REG_ITMP3);
3154
3155         /* save return value */
3156
3157         if (md->returntype.type != TYPE_VOID) {
3158                 if (IS_INT_LNG_TYPE(md->returntype.type))
3159                         M_LST(REG_RESULT, REG_SP, 0 * 8);
3160                 else
3161                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
3162         }
3163
3164 #if !defined(NDEBUG)
3165         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3166                 emit_verbosecall_exit(jd);
3167 #endif
3168
3169         /* remove native stackframe info */
3170
3171         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3172         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3173         M_CALL(REG_ITMP1);
3174         M_MOV(REG_RESULT, REG_ITMP3);
3175
3176         /* restore return value */
3177
3178         if (md->returntype.type != TYPE_VOID) {
3179                 if (IS_INT_LNG_TYPE(md->returntype.type))
3180                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
3181                 else
3182                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3183         }
3184
3185         /* remove stackframe */
3186
3187         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3188
3189         /* test for exception */
3190
3191         M_TEST(REG_ITMP3);
3192         M_BNE(1);
3193         M_RET;
3194
3195         /* handle exception */
3196
3197         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3198         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3199         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3200
3201         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3202         M_JMP(REG_ITMP3);
3203
3204         /* generate patcher stubs */
3205
3206         emit_patcher_stubs(jd);
3207
3208         codegen_finish(jd);
3209
3210         return code->entrypoint;
3211 }
3212
3213
3214 /*
3215  * These are local overrides for various environment variables in Emacs.
3216  * Please do not remove this and leave it at the end of the file, where
3217  * Emacs will automagically detect them.
3218  * ---------------------------------------------------------------------
3219  * Local variables:
3220  * mode: c
3221  * indent-tabs-mode: t
3222  * c-basic-offset: 4
3223  * tab-width: 4
3224  * End:
3225  * vim:noexpandtab:sw=4:ts=4:
3226  */