src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31             Edwin Steiner
  32
  33    $Id: codegen.c 5561 2006-09-28 19:53:05Z edwin $
  34
  35 */
  36
  37
  38 #include "config.h"
  39
  40 #include <assert.h>
  41 #include <stdio.h>
  42
  43 #include "vm/types.h"
  44
  45 #include "md.h"
  46 #include "md-abi.h"
  47
  48 #include "vm/jit/x86_64/arch.h"
  49 #include "vm/jit/x86_64/codegen.h"
  50 #include "vm/jit/x86_64/emit.h"
  51
  52 #include "mm/memory.h"
  53 #include "native/jni.h"
  54 #include "native/native.h"
  55
  56 #if defined(ENABLE_THREADS)
  57 # include "threads/native/lock.h"
  58 #endif
  59
  60 #include "vm/builtin.h"
  61 #include "vm/exceptions.h"
  62 #include "vm/global.h"
  63 #include "vm/loader.h"
  64 #include "vm/options.h"
  65 #include "vm/statistics.h"
  66 #include "vm/stringlocal.h"
  67 #include "vm/vm.h"
  68 #include "vm/jit/asmpart.h"
  69 #include "vm/jit/codegen-common.h"
  70 #include "vm/jit/dseg.h"
  71 #include "vm/jit/emit-common.h"
  72 #include "vm/jit/jit.h"
  73 #include "vm/jit/methodheader.h"
  74 #include "vm/jit/parse.h"
  75 #include "vm/jit/patcher.h"
  76 #include "vm/jit/reg.h"
  77 #include "vm/jit/replace.h"
  78
  79 #if defined(ENABLE_LSRA)
  80 # include "vm/jit/allocator/lsra.h"
  81 #endif
  82
  83
  84 /* codegen *********************************************************************
  85
  86    Generates machine code.
  87
  88 *******************************************************************************/
  89
  90 bool codegen(jitdata *jd)
  91 {
  92         methodinfo         *m;
  93         codeinfo           *code;
  94         codegendata        *cd;
  95         registerdata       *rd;
  96         s4                  len, s1, s2, s3, d, disp;
  97         u2                  currentline;
  98         ptrint              a;
  99         s4                  fieldtype;
 100         varinfo            *var, *var1, *var2, *dst;
 101         basicblock         *bptr;
 102         instruction        *iptr;
 103         exceptiontable     *ex;
 104         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 105         unresolved_method  *um;
 106         builtintable_entry *bte;
 107         methoddesc         *md;
 108         rplpoint           *replacementpoint;
 109         s4                 varindex;
 110
 111         /* get required compiler data */
 112
 113         m    = jd->m;
 114         code = jd->code;
 115         cd   = jd->cd;
 116         rd   = jd->rd;
 117
 118         /* prevent compiler warnings */
 119
 120         d = 0;
 121         lm = NULL;
 122         bte = NULL;
 123
 124         {
 125         s4 i, p, t, l;
 126         s4 savedregs_num;
 127
 128         savedregs_num = 0;
 129
 130         /* space to save used callee saved registers */
 131
 132         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 133         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 134
 135         cd->stackframesize = rd->memuse + savedregs_num;
 136
 137 #if defined(ENABLE_THREADS)
 138         /* space to save argument of monitor_enter */
 139
 140         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 141                 cd->stackframesize++;
 142 #endif
 143
 144         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 145            native code e.g. libc or jni (alignment problems with
 146            movaps). */
 147
 148         if (!jd->isleafmethod || opt_verbosecall)
 149                 cd->stackframesize |= 0x1;
 150
 151         /* create method header */
 152
 153         (void) dseg_addaddress(cd, code);                      /* CodeinfoPointer */
 154         (void) dseg_adds4(cd, cd->stackframesize * 8);         /* FrameSize       */
 155
 156 #if defined(ENABLE_THREADS)
 157         /* IsSync contains the offset relative to the stack pointer for the
 158            argument of monitor_exit used in the exception handler. Since the
 159            offset could be zero and give a wrong meaning of the flag it is
 160            offset by one.
 161         */
 162
 163         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 164                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);       /* IsSync          */
 165         else
 166 #endif
 167                 (void) dseg_adds4(cd, 0);                          /* IsSync          */
 168
 169         (void) dseg_adds4(cd, jd->isleafmethod);               /* IsLeaf          */
 170         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave         */
 171         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave         */
 172
 173         (void) dseg_addlinenumbertablesize(cd);
 174
 175         (void) dseg_adds4(cd, cd->exceptiontablelength);       /* ExTableSize     */
 176
 177         /* create exception table */
 178
 179         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 180                 dseg_addtarget(cd, ex->start);
 181                 dseg_addtarget(cd, ex->end);
 182                 dseg_addtarget(cd, ex->handler);
 183                 (void) dseg_addaddress(cd, ex->catchtype.any);
 184         }
 185
 186         /* generate method profiling code */
 187
 188         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 189                 /* count frequency */
 190
 191                 M_MOV_IMM(code, REG_ITMP3);
 192                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 193
 194                 PROFILE_CYCLE_START;
 195         }
 196
 197         /* create stack frame (if necessary) */
 198
 199         if (cd->stackframesize)
 200                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 201
 202         /* save used callee saved registers */
 203
 204         p = cd->stackframesize;
 205         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 206                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 207         }
 208         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 209                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 210         }
 211
 212         /* take arguments out of register or stack frame */
 213
 214         md = m->parseddesc;
 215
 216         for (p = 0, l = 0; p < md->paramcount; p++) {
 217                 t = md->paramtypes[p].type;
 218
 219                 varindex = jd->local_map[l * 5 + t];
 220
 221                 l++;
 222                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 223                         l++;
 224
 225                 if (varindex == UNUSED)
 226                         continue;
 227
 228                 var = jd->var + varindex;
 229
 230                 s1 = md->params[p].regoff;
 231
 232                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 233                         s2 = rd->argintregs[s1];
 234                         if (!md->params[p].inmemory) {           /* register arguments    */
 235                                 if (!IS_INMEMORY(var->flags)) {      /* reg arg -> register   */
 236                                         M_INTMOVE(s2, var->vv.regoff);
 237
 238                                 } else {                             /* reg arg -> spilled    */
 239                                     M_LST(s2, REG_SP, var->vv.regoff * 8);
 240                                 }
 241
 242                         } else {                                 /* stack arguments       */
 243                                 if (!IS_INMEMORY(var->flags)) {      /* stack arg -> register */
 244                                         /* + 8 for return address */
 245                                         M_LLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 246
 247                                 } else {                             /* stack arg -> spilled  */
 248                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 249                                 }
 250                         }
 251
 252                 } else {                                     /* floating args         */
 253                         if (!md->params[p].inmemory) {           /* register arguments    */
 254                                 s2 = rd->argfltregs[s1];
 255                                 if (!IS_INMEMORY(var->flags)) {      /* reg arg -> register   */
 256                                         M_FLTMOVE(s2, var->vv.regoff);
 257
 258                                 } else {                                         /* reg arg -> spilled    */
 259                                         M_DST(s2, REG_SP, var->vv.regoff * 8);
 260                                 }
 261
 262                         } else {                                 /* stack arguments       */
 263                                 if (!IS_INMEMORY(var->flags)) {      /* stack-arg -> register */
 264                                         M_DLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 265
 266                                 } else {
 267                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 268                                 }
 269                         }
 270                 }
 271         }  /* end for */
 272
 273         /* save monitorenter argument */
 274
 275 #if defined(ENABLE_THREADS)
 276         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 277                 /* stack offset for monitor argument */
 278
 279                 s1 = rd->memuse;
 280
 281                 if (opt_verbosecall) {
 282                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 283
 284                         for (p = 0; p < INT_ARG_CNT; p++)
 285                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 286
 287                         for (p = 0; p < FLT_ARG_CNT; p++)
 288                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 289
 290                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 291                 }
 292
 293                 /* decide which monitor enter function to call */
 294
 295                 if (m->flags & ACC_STATIC) {
 296                         M_MOV_IMM(&m->class->object.header, REG_A0);
 297                 }
 298                 else {
 299                         M_TEST(REG_A0);
 300                         M_BEQ(0);
 301                         codegen_add_nullpointerexception_ref(cd);
 302                 }
 303
 304                 M_AST(REG_A0, REG_SP, s1 * 8);
 305                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 306                 M_CALL(REG_ITMP1);
 307
 308                 if (opt_verbosecall) {
 309                         for (p = 0; p < INT_ARG_CNT; p++)
 310                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 311
 312                         for (p = 0; p < FLT_ARG_CNT; p++)
 313                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 314
 315                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 316                 }
 317         }
 318 #endif
 319
 320 #if !defined(NDEBUG)
 321         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 322                 emit_verbosecall_enter(jd);
 323 #endif /* !defined(NDEBUG) */
 324
 325         }
 326
 327         /* end of header generation */
 328
 329         replacementpoint = jd->code->rplpoints;
 330
 331         /* walk through all basic blocks */
 332
 333         for (bptr = jd->new_basicblocks; bptr != NULL; bptr = bptr->next) {
 334
 335                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 336
 337                 if (bptr->flags >= BBREACHED) {
 338
 339                         /* branch resolving */
 340
 341                         branchref *bref;
 342                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 343                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 344                                                                   bref->branchpos,
 345                                                                   bptr->mpc);
 346                         }
 347
 348                 /* handle replacement points */
 349
 350 #if 0
 351                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 352                         replacementpoint->pc = (u1*)(ptrint)bptr->mpc; /* will be resolved later */
 353
 354                         replacementpoint++;
 355
 356                         assert(cd->lastmcodeptr <= cd->mcodeptr);
 357                         cd->lastmcodeptr = cd->mcodeptr + 5; /* 5 byte jmp patch */
 358                 }
 359 #endif
 360
 361                 /* copy interface registers to their destination */
 362
 363                 len = bptr->indepth;
 364                 MCODECHECK(512);
 365
 366                 /* generate basicblock profiling code */
 367
 368                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 369                         /* count frequency */
 370
 371                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 372                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 373
 374                         /* if this is an exception handler, start profiling again */
 375
 376                         if (bptr->type == BBTYPE_EXH)
 377                                 PROFILE_CYCLE_START;
 378                 }
 379
 380 #if defined(ENABLE_LSRA)
 381                 if (opt_lsra) {
 382                         while (len) {
 383                                 len--;
 384                                 src = bptr->invars[len];
 385                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 386                                         if (bptr->type == BBTYPE_SBR) {
 387 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 388                                                 if (!IS_INMEMORY(src->flags))
 389                                                         d = src->vv.regoff;
 390                                                 else
 391                                                         d = REG_ITMP1;
 392                                                 M_POP(d);
 393                                                 emit_store(jd, NULL, src, d);
 394
 395                                         } else if (bptr->type == BBTYPE_EXH) {
 396 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 397                                                 if (!IS_INMEMORY(src->flags))
 398                                                         d= src->vv.regoff;
 399                                                 else
 400                                                         d=REG_ITMP1;
 401                                                 M_INTMOVE(REG_ITMP1, d);
 402                                                 emit_store(jd, NULL, src, d);
 403                                         }
 404                                 }
 405                         }
 406
 407                 } else {
 408 #endif
 409
 410                 while (len) {
 411                         len--;
 412                         varindex = bptr->invars[len];
 413                         var = jd->var + varindex;
 414                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 415                                 if (bptr->type == BBTYPE_SBR) {
 416                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 417                                         M_POP(d);
 418                                         emit_store(jd, NULL, var, d);
 419
 420                                 }
 421                                 else if (bptr->type == BBTYPE_EXH) {
 422                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 423                                         M_INTMOVE(REG_ITMP1, d);
 424                                         emit_store(jd, NULL, var, d);
 425                                 }
 426
 427                         } else {
 428                                 assert((var->flags & OUTVAR));
 429                                 d = codegen_reg_of_var(0, var, REG_ITMP1);
 430                         }
 431                 }
 432 #if defined(ENABLE_LSRA)
 433                 }
 434 #endif
 435                 /* walk through all instructions */
 436
 437                 len = bptr->icount;
 438                 currentline = 0;
 439
 440                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 441                         if (iptr->line != currentline) {
 442                                 dseg_addlinenumber(cd, iptr->line);
 443                                 currentline = iptr->line;
 444                         }
 445
 446                         MCODECHECK(1024);                         /* 1KB should be enough */
 447
 448                         switch (iptr->opc) {
 449                         case ICMD_INLINE_START: /* internal ICMDs                         */
 450                         case ICMD_INLINE_END:
 451                                 break;
 452
 453                         case ICMD_NOP:    /* ...  ==> ...                                 */
 454                                 break;
 455
 456                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 457
 458                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 459                                 M_TEST(s1);
 460                                 M_BEQ(0);
 461                                 codegen_add_nullpointerexception_ref(cd);
 462                                 break;
 463
 464                 /* constant operations ************************************************/
 465
 466                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 467
 468                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 469                         ICONST(d, iptr->sx.val.i);
 470                         emit_store_dst(jd, iptr, d);
 471                         break;
 472
 473                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 474
 475                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 476                         LCONST(d, iptr->sx.val.l);
 477                         emit_store_dst(jd, iptr, d);
 478                         break;
 479
 480                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 481
 482                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 483                         disp = dseg_addfloat(cd, iptr->sx.val.f);
 484                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 485                         emit_store_dst(jd, iptr, d);
 486                         break;
 487
 488                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 489
 490                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 491                         disp = dseg_adddouble(cd, iptr->sx.val.d);
 492                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 493                         emit_store_dst(jd, iptr, d);
 494                         break;
 495
 496                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 497
 498                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 499
 500                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 501 /*                              PROFILE_CYCLE_STOP; */
 502
 503                                 codegen_addpatchref(cd, PATCHER_aconst,
 504                                                                         iptr->sx.val.c.ref, 0);
 505
 506                                 if (opt_showdisassemble) {
 507                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
 508                                 }
 509
 510 /*                              PROFILE_CYCLE_START; */
 511
 512                                 M_MOV_IMM(NULL, d);
 513
 514                         } else {
 515                                 if (iptr->sx.val.anyptr == 0)
 516                                         M_CLR(d);
 517                                 else
 518                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 519                         }
 520                         emit_store_dst(jd, iptr, d);
 521                         break;
 522
 523
 524                 /* load/store operations **********************************************/
 525
 526                 case ICMD_ILOAD:
 527                 case ICMD_ALOAD:
 528                 case ICMD_LLOAD:
 529                 case ICMD_FLOAD:
 530                 case ICMD_DLOAD:
 531                 case ICMD_ISTORE:
 532                 case ICMD_ASTORE:
 533                 case ICMD_LSTORE:
 534                 case ICMD_FSTORE:
 535                 case ICMD_DSTORE:
 536
 537                         emit_copy(jd, iptr, jd->var + iptr->s1.varindex,
 538                                                                 jd->var + iptr->dst.varindex);
 539                         break;
 540
 541                 /* pop/copy/move operations *******************************************/
 542
 543                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 544
 545                 case ICMD_POP:        /* ..., value  ==> ...                          */
 546                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 547
 548                         break;
 549
 550                 case ICMD_COPY:
 551                 case ICMD_MOVE:
 552
 553                         M_COPY(iptr->s1.varindex, iptr->dst.varindex);
 554                         break;
 555
 556                 /* integer operations *************************************************/
 557
 558                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 559
 560                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 561                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 562                         M_INTMOVE(s1, d);
 563                         M_INEG(d);
 564                         emit_store_dst(jd, iptr, d);
 565                         break;
 566
 567                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 568
 569                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 570                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 571                         M_INTMOVE(s1, d);
 572                         M_LNEG(d);
 573                         emit_store_dst(jd, iptr, d);
 574                         break;
 575
 576                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 577
 578                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 579                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 580                         M_ISEXT(s1, d);
 581                         emit_store_dst(jd, iptr, d);
 582                         break;
 583
 584                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 585
 586                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 587                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 588                         M_IMOV(s1, d);
 589                         emit_store_dst(jd, iptr, d);
 590                         break;
 591
 592                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 593
 594                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 595                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 596                         M_BSEXT(s1, d);
 597                         emit_store_dst(jd, iptr, d);
 598                         break;
 599
 600                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 601
 602                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 603                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 604                         M_CZEXT(s1, d);
 605                         emit_store_dst(jd, iptr, d);
 606                         break;
 607
 608                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 609
 610                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 611                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 612                         M_SSEXT(s1, d);
 613                         emit_store_dst(jd, iptr, d);
 614                         break;
 615
 616
 617                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 618
 619                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 620                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 621                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 622                         if (s2 == d)
 623                                 M_IADD(s1, d);
 624                         else {
 625                                 M_INTMOVE(s1, d);
 626                                 M_IADD(s2, d);
 627                         }
 628                         emit_store_dst(jd, iptr, d);
 629                         break;
 630
 631                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 632                                       /* sx.val.i = constant                             */
 633
 634                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 635                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 636                         M_INTMOVE(s1, d);
 637                         M_IADD_IMM(iptr->sx.val.i, d);
 638                         emit_store_dst(jd, iptr, d);
 639                         break;
 640
 641                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 642
 643                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 644                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 645                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 646                         if (s2 == d)
 647                                 M_LADD(s1, d);
 648                         else {
 649                                 M_INTMOVE(s1, d);
 650                                 M_LADD(s2, d);
 651                         }
 652                         emit_store_dst(jd, iptr, d);
 653                         break;
 654
 655                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 656                                       /* sx.val.l = constant                             */
 657
 658                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 659                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 660                         M_INTMOVE(s1, d);
 661                         if (IS_IMM32(iptr->sx.val.l))
 662                                 M_LADD_IMM(iptr->sx.val.l, d);
 663                         else {
 664                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 665                                 M_LADD(REG_ITMP2, d);
 666                         }
 667                         emit_store_dst(jd, iptr, d);
 668                         break;
 669
 670                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 671
 672                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 673                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 674                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 675                         if (s2 == d) {
 676                                 M_INTMOVE(s1, REG_ITMP1);
 677                                 M_ISUB(s2, REG_ITMP1);
 678                                 M_INTMOVE(REG_ITMP1, d);
 679                         } else {
 680                                 M_INTMOVE(s1, d);
 681                                 M_ISUB(s2, d);
 682                         }
 683                         emit_store_dst(jd, iptr, d);
 684                         break;
 685
 686                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 687                                       /* sx.val.i = constant                             */
 688
 689                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 690                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 691                         M_INTMOVE(s1, d);
 692                         M_ISUB_IMM(iptr->sx.val.i, d);
 693                         emit_store_dst(jd, iptr, d);
 694                         break;
 695
 696                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 697
 698                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 699                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 700                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 701                         if (s2 == d) {
 702                                 M_INTMOVE(s1, REG_ITMP1);
 703                                 M_LSUB(s2, REG_ITMP1);
 704                                 M_INTMOVE(REG_ITMP1, d);
 705                         } else {
 706                                 M_INTMOVE(s1, d);
 707                                 M_LSUB(s2, d);
 708                         }
 709                         emit_store_dst(jd, iptr, d);
 710                         break;
 711
 712                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 713                                       /* sx.val.l = constant                             */
 714
 715                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 716                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 717                         M_INTMOVE(s1, d);
 718                         if (IS_IMM32(iptr->sx.val.l))
 719                                 M_LSUB_IMM(iptr->sx.val.l, d);
 720                         else {
 721                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 722                                 M_LSUB(REG_ITMP2, d);
 723                         }
 724                         emit_store_dst(jd, iptr, d);
 725                         break;
 726
 727                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 728
 729                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 730                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 731                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 732                         if (s2 == d)
 733                                 M_IMUL(s1, d);
 734                         else {
 735                                 M_INTMOVE(s1, d);
 736                                 M_IMUL(s2, d);
 737                         }
 738                         emit_store_dst(jd, iptr, d);
 739                         break;
 740
 741                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 742                                       /* sx.val.i = constant                             */
 743
 744                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 745                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 746                         if (iptr->sx.val.i == 2) {
 747                                 M_INTMOVE(s1, d);
 748                                 M_ISLL_IMM(1, d);
 749                         } else
 750                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 751                         emit_store_dst(jd, iptr, d);
 752                         break;
 753
 754                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 755
 756                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 757                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 758                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 759                         if (s2 == d)
 760                                 M_LMUL(s1, d);
 761                         else {
 762                                 M_INTMOVE(s1, d);
 763                                 M_LMUL(s2, d);
 764                         }
 765                         emit_store_dst(jd, iptr, d);
 766                         break;
 767
 768                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 769                                       /* sx.val.l = constant                             */
 770
 771                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 772                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 773                         if (IS_IMM32(iptr->sx.val.l))
 774                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 775                         else {
 776                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 777                                 M_INTMOVE(s1, d);
 778                                 M_LMUL(REG_ITMP2, d);
 779                         }
 780                         emit_store_dst(jd, iptr, d);
 781                         break;
 782
 783                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 784
 785                         var1 = jd->var + iptr->s1.varindex;
 786                         var2 = jd->var + iptr->sx.s23.s2.varindex;
 787                         dst  = jd->var + iptr->dst.varindex;
 788
 789                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 790                 if (IS_INMEMORY(var1->flags))
 791                                 M_ILD(RAX, REG_SP, var1->vv.regoff * 8);
 792                         else
 793                                 M_INTMOVE(var1->vv.regoff, RAX);
 794
 795                         if (IS_INMEMORY(var2->flags))
 796                                 M_ILD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 797                         else
 798                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 799
 800                         if (checknull) {
 801                                 M_ITEST(REG_ITMP3);
 802                                 M_BEQ(0);
 803                                 codegen_add_arithmeticexception_ref(cd);
 804                         }
 805
 806                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
 807                         emit_jcc(cd, CC_NE, 4 + 6);
 808                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
 809                         emit_jcc(cd, CC_E, 3 + 1 + 3);                /* 6 bytes */
 810
 811                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 812                         emit_cltd(cd);
 813                         emit_idivl_reg(cd, REG_ITMP3);
 814
 815                         if (IS_INMEMORY(dst->flags)) {
 816                                 emit_mov_reg_membase(cd, RAX, REG_SP, dst->vv.regoff * 8);
 817                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
 818
 819                         } else {
 820                                 M_INTMOVE(RAX, dst->vv.regoff);
 821
 822                                 if (dst->vv.regoff != RDX) {
 823                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
 824                                 }
 825                         }
 826                         break;
 827
 828                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 829
 830                         var1 = jd->var + iptr->s1.varindex;
 831                         var2 = jd->var + iptr->sx.s23.s2.varindex;
 832                         dst  = jd->var + iptr->dst.varindex;
 833
 834                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 835                         if (IS_INMEMORY(var1->flags))
 836                                 M_ILD(RAX, REG_SP, var1->vv.regoff * 8);
 837                         else
 838                                 M_INTMOVE(var1->vv.regoff, RAX);
 839
 840                         if (IS_INMEMORY(var2->flags))
 841                                 M_ILD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 842                         else
 843                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 844
 845                         if (checknull) {
 846                                 M_ITEST(REG_ITMP3);
 847                                 M_BEQ(0);
 848                                 codegen_add_arithmeticexception_ref(cd);
 849                         }
 850
 851                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 852
 853                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
 854                         emit_jcc(cd, CC_NE, 2 + 4 + 6);
 855
 856
 857                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
 858                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
 859                         emit_jcc(cd, CC_E, 1 + 3);                    /* 6 bytes */
 860
 861                         emit_cltd(cd);
 862                         emit_idivl_reg(cd, REG_ITMP3);
 863
 864                         if (IS_INMEMORY(dst->flags)) {
 865                                 emit_mov_reg_membase(cd, RDX, REG_SP, dst->vv.regoff * 8);
 866                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
 867
 868                         } else {
 869                                 M_INTMOVE(RDX, dst->vv.regoff);
 870
 871                                 if (dst->vv.regoff != RDX) {
 872                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
 873                                 }
 874                         }
 875                         break;
 876
 877                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 878                                       /* sx.val.i = constant                             */
 879
 880                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 881                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 882                         M_INTMOVE(s1, REG_ITMP1);
 883                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 884                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 885                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 886                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 887                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 888                         emit_store_dst(jd, iptr, d);
 889                         break;
 890
 891                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 892                                       /* sx.val.i = constant                             */
 893
 894                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 895                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 896                         M_INTMOVE(s1, REG_ITMP1);
 897                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 898                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 899                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 900                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 901                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 902                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 903                         emit_store_dst(jd, iptr, d);
 904                         break;
 905
 906
 907                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 908
 909                         var1 = jd->var + iptr->s1.varindex;
 910                         var2 = jd->var + iptr->sx.s23.s2.varindex;
 911                         dst  = jd->var + iptr->dst.varindex;
 912
 913                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 914
 915                 if (IS_INMEMORY(var1->flags))
 916                                 M_LLD(RAX, REG_SP, var1->vv.regoff * 8);
 917                         else
 918                                 M_INTMOVE(var1->vv.regoff, RAX);
 919
 920                         if (IS_INMEMORY(var2->flags))
 921                                 M_LLD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 922                         else
 923                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 924
 925                         if (checknull) {
 926                                 M_TEST(REG_ITMP3);
 927                                 M_BEQ(0);
 928                                 codegen_add_arithmeticexception_ref(cd);
 929                         }
 930
 931                         /* check as described in jvm spec */
 932                         disp = dseg_adds8(cd, 0x8000000000000000LL);
 933                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 934                         M_BNE(4 + 6);
 935                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
 936                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
 937
 938                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 939                         emit_cqto(cd);
 940                         emit_idiv_reg(cd, REG_ITMP3);
 941
 942                         if (IS_INMEMORY(dst->flags)) {
 943                                 M_LST(RAX, REG_SP, dst->vv.regoff * 8);
 944                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
 945
 946                         } else {
 947                                 M_INTMOVE(RAX, dst->vv.regoff);
 948
 949                                 if (dst->vv.regoff != RDX) {
 950                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
 951                                 }
 952                         }
 953                         break;
 954
 955                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 956
 957                         var1 = jd->var + iptr->s1.varindex;
 958                         var2 = jd->var + iptr->sx.s23.s2.varindex;
 959                         dst  = jd->var + iptr->dst.varindex;
 960
 961                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 962
 963                         if (IS_INMEMORY(var1->flags))
 964                                 M_LLD(REG_ITMP1, REG_SP, var1->vv.regoff * 8);
 965                         else
 966                                 M_INTMOVE(var1->vv.regoff, REG_ITMP1);
 967
 968                         if (IS_INMEMORY(var2->flags))
 969                                 M_LLD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 970                         else
 971                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 972                         /*
 973                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 974                         M_INTMOVE(s1, REG_ITMP1);
 975                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 976                         M_INTMOVE(s2, REG_ITMP3);
 977                         */
 978                         if (checknull) {
 979                                 M_ITEST(REG_ITMP3);
 980                                 M_BEQ(0);
 981                                 codegen_add_arithmeticexception_ref(cd);
 982                         }
 983
 984                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 985
 986                         /* check as described in jvm spec */
 987                         disp = dseg_adds8(cd, 0x8000000000000000LL);
 988                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 989                         M_BNE(3 + 4 + 6);
 990
 991 #if 0
 992                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
 993 #endif
 994                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 995                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
 996                         M_BEQ(2 + 3);                                          /* 6 bytes */
 997
 998                         emit_cqto(cd);
 999                         emit_idiv_reg(cd, REG_ITMP3);
1000
1001                         if (IS_INMEMORY(dst->flags)) {
1002                                 M_LST(RDX, REG_SP, dst->vv.regoff * 8);
1003                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1004
1005                         } else {
1006                                 M_INTMOVE(RDX, dst->vv.regoff);
1007
1008                                 if (dst->vv.regoff != RDX) {
1009                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1010                                 }
1011                         }
1012                         break;
1013
1014                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1015                                       /* sx.val.i = constant                             */
1016
1017                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1018                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1019                         M_INTMOVE(s1, REG_ITMP1);
1020                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1021                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
1022                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
1023                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
1024                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1025                         emit_store_dst(jd, iptr, d);
1026                         break;
1027
1028                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1029                                       /* sx.val.l = constant                             */
1030
1031                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1032                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1033                         M_INTMOVE(s1, REG_ITMP1);
1034                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1035                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
1036                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
1037                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
1038                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
1039                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1040                         emit_store_dst(jd, iptr, d);
1041                         break;
1042
1043                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1044
1045                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1046                         emit_ishift(jd, SHIFT_SHL, iptr);
1047                         break;
1048
1049                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1050                                       /* sx.val.i = constant                             */
1051
1052                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1053                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1054                         M_INTMOVE(s1, d);
1055                         M_ISLL_IMM(iptr->sx.val.i, d);
1056                         emit_store_dst(jd, iptr, d);
1057                         break;
1058
1059                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1060
1061                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1062                         emit_ishift(jd, SHIFT_SAR, iptr);
1063                         break;
1064
1065                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1066                                       /* sx.val.i = constant                             */
1067
1068                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1069                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1070                         M_INTMOVE(s1, d);
1071                         M_ISRA_IMM(iptr->sx.val.i, d);
1072                         emit_store_dst(jd, iptr, d);
1073                         break;
1074
1075                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1076
1077                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1078                         emit_ishift(jd, SHIFT_SHR, iptr);
1079                         break;
1080
1081                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1082                                       /* sx.val.i = constant                             */
1083
1084                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1085                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1086                         M_INTMOVE(s1, d);
1087                         M_ISRL_IMM(iptr->sx.val.i, d);
1088                         emit_store_dst(jd, iptr, d);
1089                         break;
1090
1091                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1092
1093                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1094                         emit_lshift(jd, SHIFT_SHL, iptr);
1095                         break;
1096
1097         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1098                                           /* sx.val.i = constant                             */
1099
1100                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1101                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1102                         M_INTMOVE(s1, d);
1103                         M_LSLL_IMM(iptr->sx.val.i, d);
1104                         emit_store_dst(jd, iptr, d);
1105                         break;
1106
1107                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1108
1109                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1110                         emit_lshift(jd, SHIFT_SAR, iptr);
1111                         break;
1112
1113                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1114                                       /* sx.val.i = constant                             */
1115
1116                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1117                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1118                         M_INTMOVE(s1, d);
1119                         M_LSRA_IMM(iptr->sx.val.i, d);
1120                         emit_store_dst(jd, iptr, d);
1121                         break;
1122
1123                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1124
1125                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1126                         emit_lshift(jd, SHIFT_SHR, iptr);
1127                         break;
1128
1129                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1130                                       /* sx.val.l = constant                             */
1131
1132                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1133                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1134                         M_INTMOVE(s1, d);
1135                         M_LSRL_IMM(iptr->sx.val.i, d);
1136                         emit_store_dst(jd, iptr, d);
1137                         break;
1138
1139                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1140
1141                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1142                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1143                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1144                         if (s2 == d)
1145                                 M_IAND(s1, d);
1146                         else {
1147                                 M_INTMOVE(s1, d);
1148                                 M_IAND(s2, d);
1149                         }
1150                         emit_store_dst(jd, iptr, d);
1151                         break;
1152
1153                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1154                                       /* sx.val.i = constant                             */
1155
1156                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1157                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1158                         M_INTMOVE(s1, d);
1159                         M_IAND_IMM(iptr->sx.val.i, d);
1160                         emit_store_dst(jd, iptr, d);
1161                         break;
1162
1163                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1164
1165                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1166                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1167                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1168                         if (s2 == d)
1169                                 M_LAND(s1, d);
1170                         else {
1171                                 M_INTMOVE(s1, d);
1172                                 M_LAND(s2, d);
1173                         }
1174                         emit_store_dst(jd, iptr, d);
1175                         break;
1176
1177                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1178                                       /* sx.val.l = constant                             */
1179
1180                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1181                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1182                         M_INTMOVE(s1, d);
1183                         if (IS_IMM32(iptr->sx.val.l))
1184                                 M_LAND_IMM(iptr->sx.val.l, d);
1185                         else {
1186                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1187                                 M_LAND(REG_ITMP2, d);
1188                         }
1189                         emit_store_dst(jd, iptr, d);
1190                         break;
1191
1192                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1193
1194                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1195                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1196                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1197                         if (s2 == d)
1198                                 M_IOR(s1, d);
1199                         else {
1200                                 M_INTMOVE(s1, d);
1201                                 M_IOR(s2, d);
1202                         }
1203                         emit_store_dst(jd, iptr, d);
1204                         break;
1205
1206                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1207                                       /* sx.val.i = constant                             */
1208
1209                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1210                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1211                         M_INTMOVE(s1, d);
1212                         M_IOR_IMM(iptr->sx.val.i, d);
1213                         emit_store_dst(jd, iptr, d);
1214                         break;
1215
1216                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1217
1218                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1219                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1220                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1221                         if (s2 == d)
1222                                 M_LOR(s1, d);
1223                         else {
1224                                 M_INTMOVE(s1, d);
1225                                 M_LOR(s2, d);
1226                         }
1227                         emit_store_dst(jd, iptr, d);
1228                         break;
1229
1230                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1231                                       /* sx.val.l = constant                             */
1232
1233                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1234                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1235                         M_INTMOVE(s1, d);
1236                         if (IS_IMM32(iptr->sx.val.l))
1237                                 M_LOR_IMM(iptr->sx.val.l, d);
1238                         else {
1239                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1240                                 M_LOR(REG_ITMP2, d);
1241                         }
1242                         emit_store_dst(jd, iptr, d);
1243                         break;
1244
1245                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1246
1247                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1248                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1249                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1250                         if (s2 == d)
1251                                 M_IXOR(s1, d);
1252                         else {
1253                                 M_INTMOVE(s1, d);
1254                                 M_IXOR(s2, d);
1255                         }
1256                         emit_store_dst(jd, iptr, d);
1257                         break;
1258
1259                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1260                                       /* sx.val.i = constant                             */
1261
1262                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1263                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1264                         M_INTMOVE(s1, d);
1265                         M_IXOR_IMM(iptr->sx.val.i, d);
1266                         emit_store_dst(jd, iptr, d);
1267                         break;
1268
1269                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1270
1271                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1272                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1273                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1274                         if (s2 == d)
1275                                 M_LXOR(s1, d);
1276                         else {
1277                                 M_INTMOVE(s1, d);
1278                                 M_LXOR(s2, d);
1279                         }
1280                         emit_store_dst(jd, iptr, d);
1281                         break;
1282
1283                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1284                                       /* sx.val.l = constant                             */
1285
1286                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1287                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1288                         M_INTMOVE(s1, d);
1289                         if (IS_IMM32(iptr->sx.val.l))
1290                                 M_LXOR_IMM(iptr->sx.val.l, d);
1291                         else {
1292                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1293                                 M_LXOR(REG_ITMP2, d);
1294                         }
1295                         emit_store_dst(jd, iptr, d);
1296                         break;
1297
1298
1299                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1300                                       /* s1.localindex = variable, sx.val.i = constant             */
1301
1302                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1303                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1304
1305                         /* Using inc and dec is not faster than add (tested with
1306                            sieve). */
1307
1308                         M_INTMOVE(s1, d);
1309                         M_IADD_IMM(iptr->sx.val.i, d);
1310
1311                         emit_store_dst(jd, iptr, d);
1312                         break;
1313
1314
1315                 /* floating operations ************************************************/
1316
1317                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1318
1319                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1320                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1321                         disp = dseg_adds4(cd, 0x80000000);
1322                         M_FLTMOVE(s1, d);
1323                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1324                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1325                         emit_store_dst(jd, iptr, d);
1326                         break;
1327
1328                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1329
1330                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1331                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1332                         disp = dseg_adds8(cd, 0x8000000000000000);
1333                         M_FLTMOVE(s1, d);
1334                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1335                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1336                         emit_store_dst(jd, iptr, d);
1337                         break;
1338
1339                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1340
1341                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1342                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1343                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1344                         if (s2 == d)
1345                                 M_FADD(s1, d);
1346                         else {
1347                                 M_FLTMOVE(s1, d);
1348                                 M_FADD(s2, d);
1349                         }
1350                         emit_store_dst(jd, iptr, d);
1351                         break;
1352
1353                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1354
1355                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1356                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1357                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1358                         if (s2 == d)
1359                                 M_DADD(s1, d);
1360                         else {
1361                                 M_FLTMOVE(s1, d);
1362                                 M_DADD(s2, d);
1363                         }
1364                         emit_store_dst(jd, iptr, d);
1365                         break;
1366
1367                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1368
1369                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1370                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1371                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1372                         if (s2 == d) {
1373                                 M_FLTMOVE(s2, REG_FTMP2);
1374                                 s2 = REG_FTMP2;
1375                         }
1376                         M_FLTMOVE(s1, d);
1377                         M_FSUB(s2, d);
1378                         emit_store_dst(jd, iptr, d);
1379                         break;
1380
1381                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1382
1383                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1384                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1385                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1386                         if (s2 == d) {
1387                                 M_FLTMOVE(s2, REG_FTMP2);
1388                                 s2 = REG_FTMP2;
1389                         }
1390                         M_FLTMOVE(s1, d);
1391                         M_DSUB(s2, d);
1392                         emit_store_dst(jd, iptr, d);
1393                         break;
1394
1395                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1396
1397                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1398                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1399                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1400                         if (s2 == d)
1401                                 M_FMUL(s1, d);
1402                         else {
1403                                 M_FLTMOVE(s1, d);
1404                                 M_FMUL(s2, d);
1405                         }
1406                         emit_store_dst(jd, iptr, d);
1407                         break;
1408
1409                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1410
1411                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1412                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1413                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1414                         if (s2 == d)
1415                                 M_DMUL(s1, d);
1416                         else {
1417                                 M_FLTMOVE(s1, d);
1418                                 M_DMUL(s2, d);
1419                         }
1420                         emit_store_dst(jd, iptr, d);
1421                         break;
1422
1423                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1424
1425                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1426                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1427                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1428                         if (s2 == d) {
1429                                 M_FLTMOVE(s2, REG_FTMP2);
1430                                 s2 = REG_FTMP2;
1431                         }
1432                         M_FLTMOVE(s1, d);
1433                         M_FDIV(s2, d);
1434                         emit_store_dst(jd, iptr, d);
1435                         break;
1436
1437                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1438
1439                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1440                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1441                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1442                         if (s2 == d) {
1443                                 M_FLTMOVE(s2, REG_FTMP2);
1444                                 s2 = REG_FTMP2;
1445                         }
1446                         M_FLTMOVE(s1, d);
1447                         M_DDIV(s2, d);
1448                         emit_store_dst(jd, iptr, d);
1449                         break;
1450
1451                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1452
1453                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1454                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1455                         M_CVTIF(s1, d);
1456                         emit_store_dst(jd, iptr, d);
1457                         break;
1458
1459                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1460
1461                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1462                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1463                         M_CVTID(s1, d);
1464                         emit_store_dst(jd, iptr, d);
1465                         break;
1466
1467                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1468
1469                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1470                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1471                         M_CVTLF(s1, d);
1472                         emit_store_dst(jd, iptr, d);
1473                         break;
1474
1475                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1476
1477                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1478                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1479                         M_CVTLD(s1, d);
1480                         emit_store_dst(jd, iptr, d);
1481                         break;
1482
1483                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1484
1485                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1486                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1487                         M_CVTFI(s1, d);
1488                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1489                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1490                                 ((REG_RESULT == d) ? 0 : 3);
1491                         M_BNE(disp);
1492                         M_FLTMOVE(s1, REG_FTMP1);
1493                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1494                         M_CALL(REG_ITMP2);
1495                         M_INTMOVE(REG_RESULT, d);
1496                         emit_store_dst(jd, iptr, d);
1497                         break;
1498
1499                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1500
1501                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1502                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1503                         M_CVTDI(s1, d);
1504                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1505                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1506                                 ((REG_RESULT == d) ? 0 : 3);
1507                         M_BNE(disp);
1508                         M_FLTMOVE(s1, REG_FTMP1);
1509                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1510                         M_CALL(REG_ITMP2);
1511                         M_INTMOVE(REG_RESULT, d);
1512                         emit_store_dst(jd, iptr, d);
1513                         break;
1514
1515                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1516
1517                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1518                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1519                         M_CVTFL(s1, d);
1520                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1521                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1522                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1523                                 ((REG_RESULT == d) ? 0 : 3);
1524                         M_BNE(disp);
1525                         M_FLTMOVE(s1, REG_FTMP1);
1526                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1527                         M_CALL(REG_ITMP2);
1528                         M_INTMOVE(REG_RESULT, d);
1529                         emit_store_dst(jd, iptr, d);
1530                         break;
1531
1532                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1533
1534                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1535                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1536                         M_CVTDL(s1, d);
1537                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1538                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1539                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1540                                 ((REG_RESULT == d) ? 0 : 3);
1541                         M_BNE(disp);
1542                         M_FLTMOVE(s1, REG_FTMP1);
1543                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1544                         M_CALL(REG_ITMP2);
1545                         M_INTMOVE(REG_RESULT, d);
1546                         emit_store_dst(jd, iptr, d);
1547                         break;
1548
1549                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1550
1551                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1552                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1553                         M_CVTFD(s1, d);
1554                         emit_store_dst(jd, iptr, d);
1555                         break;
1556
1557                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1558
1559                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1560                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1561                         M_CVTDF(s1, d);
1562                         emit_store_dst(jd, iptr, d);
1563                         break;
1564
1565                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1566                                           /* == => 0, < => 1, > => -1 */
1567
1568                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1569                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1570                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1571                         M_CLR(d);
1572                         M_MOV_IMM(1, REG_ITMP1);
1573                         M_MOV_IMM(-1, REG_ITMP2);
1574                         emit_ucomiss_reg_reg(cd, s1, s2);
1575                         M_CMOVB(REG_ITMP1, d);
1576                         M_CMOVA(REG_ITMP2, d);
1577                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1578                         emit_store_dst(jd, iptr, d);
1579                         break;
1580
1581                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1582                                           /* == => 0, < => 1, > => -1 */
1583
1584                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1585                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1586                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1587                         M_CLR(d);
1588                         M_MOV_IMM(1, REG_ITMP1);
1589                         M_MOV_IMM(-1, REG_ITMP2);
1590                         emit_ucomiss_reg_reg(cd, s1, s2);
1591                         M_CMOVB(REG_ITMP1, d);
1592                         M_CMOVA(REG_ITMP2, d);
1593                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1594                         emit_store_dst(jd, iptr, d);
1595                         break;
1596
1597                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1598                                           /* == => 0, < => 1, > => -1 */
1599
1600                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1601                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1602                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1603                         M_CLR(d);
1604                         M_MOV_IMM(1, REG_ITMP1);
1605                         M_MOV_IMM(-1, REG_ITMP2);
1606                         emit_ucomisd_reg_reg(cd, s1, s2);
1607                         M_CMOVB(REG_ITMP1, d);
1608                         M_CMOVA(REG_ITMP2, d);
1609                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1610                         emit_store_dst(jd, iptr, d);
1611                         break;
1612
1613                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1614                                           /* == => 0, < => 1, > => -1 */
1615
1616                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1617                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1618                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1619                         M_CLR(d);
1620                         M_MOV_IMM(1, REG_ITMP1);
1621                         M_MOV_IMM(-1, REG_ITMP2);
1622                         emit_ucomisd_reg_reg(cd, s1, s2);
1623                         M_CMOVB(REG_ITMP1, d);
1624                         M_CMOVA(REG_ITMP2, d);
1625                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1626                         emit_store_dst(jd, iptr, d);
1627                         break;
1628
1629
1630                 /* memory operations **************************************************/
1631
1632                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1633
1634                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1635                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1636                         gen_nullptr_check(s1);
1637                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1638                         emit_store_dst(jd, iptr, d);
1639                         break;
1640
1641                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1642
1643                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1644                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1645                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1646                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1647                                 gen_nullptr_check(s1);
1648                                 gen_bound_check;
1649                         }
1650                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1651                         emit_store_dst(jd, iptr, d);
1652                         break;
1653
1654                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1655
1656                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1657                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1658                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1659                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1660                                 gen_nullptr_check(s1);
1661                                 gen_bound_check;
1662                         }
1663                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1664                         emit_store_dst(jd, iptr, d);
1665                         break;
1666
1667                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1668
1669                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1670                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1671                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1672                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1673                                 gen_nullptr_check(s1);
1674                                 gen_bound_check;
1675                         }
1676                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1677                         emit_store_dst(jd, iptr, d);
1678                         break;
1679
1680                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1681
1682                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1683                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1684                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1685                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1686                                 gen_nullptr_check(s1);
1687                                 gen_bound_check;
1688                         }
1689                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1690                         emit_store_dst(jd, iptr, d);
1691                         break;
1692
1693                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1694
1695                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1696                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1697                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1698                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1699                                 gen_nullptr_check(s1);
1700                                 gen_bound_check;
1701                         }
1702                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1703                         emit_store_dst(jd, iptr, d);
1704                         break;
1705
1706                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1707
1708                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1709                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1710                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1711                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1712                                 gen_nullptr_check(s1);
1713                                 gen_bound_check;
1714                         }
1715                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1716                         emit_store_dst(jd, iptr, d);
1717                         break;
1718
1719                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1720
1721                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1722                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1723                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1724                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1725                                 gen_nullptr_check(s1);
1726                                 gen_bound_check;
1727                         }
1728                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1729                         emit_store_dst(jd, iptr, d);
1730                         break;
1731
1732                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1733
1734                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1735                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1736                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1737                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1738                                 gen_nullptr_check(s1);
1739                                 gen_bound_check;
1740                         }
1741                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1742                         emit_store_dst(jd, iptr, d);
1743                         break;
1744
1745
1746                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1747
1748                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1749                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1750                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1751                                 gen_nullptr_check(s1);
1752                                 gen_bound_check;
1753                         }
1754                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1755                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1756                         break;
1757
1758                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1759
1760                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1761                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1762                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1763                                 gen_nullptr_check(s1);
1764                                 gen_bound_check;
1765                         }
1766                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1767                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1768                         break;
1769
1770                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1771
1772                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1773                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1774                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1775                                 gen_nullptr_check(s1);
1776                                 gen_bound_check;
1777                         }
1778                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1779                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1780                         break;
1781
1782                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1783
1784                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1785                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1786                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1787                                 gen_nullptr_check(s1);
1788                                 gen_bound_check;
1789                         }
1790                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1791                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
1792                         break;
1793
1794                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1795
1796                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1797                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1798                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1799                                 gen_nullptr_check(s1);
1800                                 gen_bound_check;
1801                         }
1802                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1803                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
1804                         break;
1805
1806                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1807
1808                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1809                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1810                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1811                                 gen_nullptr_check(s1);
1812                                 gen_bound_check;
1813                         }
1814                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1815                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
1816                         break;
1817
1818                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1819
1820                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1821                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1822                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1823                                 gen_nullptr_check(s1);
1824                                 gen_bound_check;
1825                         }
1826                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1827                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
1828                         break;
1829
1830                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1831
1832                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1833                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1834                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1835                                 gen_nullptr_check(s1);
1836                                 gen_bound_check;
1837                         }
1838                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1839
1840                         M_MOV(s1, REG_A0);
1841                         M_MOV(s3, REG_A1);
1842                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
1843                         M_CALL(REG_ITMP1);
1844                         M_TEST(REG_RESULT);
1845                         M_BEQ(0);
1846                         codegen_add_arraystoreexception_ref(cd);
1847
1848                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1849                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1850                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1851                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1852                         break;
1853
1854
1855                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1856
1857                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1858                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1859                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1860                                 gen_nullptr_check(s1);
1861                                 gen_bound_check;
1862                         }
1863                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1864                         break;
1865
1866                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1867
1868                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1869                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1870                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1871                                 gen_nullptr_check(s1);
1872                                 gen_bound_check;
1873                         }
1874                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray, data[0]), s1, s2, 1);
1875                         break;
1876
1877                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1878
1879                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1880                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1881                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1882                                 gen_nullptr_check(s1);
1883                                 gen_bound_check;
1884                         }
1885                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1886                         break;
1887
1888                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1889
1890                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1891                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1892                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1893                                 gen_nullptr_check(s1);
1894                                 gen_bound_check;
1895                         }
1896                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray, data[0]), s1, s2, 2);
1897                         break;
1898
1899                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1900
1901                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1902                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1903                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1904                                 gen_nullptr_check(s1);
1905                                 gen_bound_check;
1906                         }
1907
1908                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1909                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1910                         } else {
1911                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1912                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
1913                         }
1914                         break;
1915
1916                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1917
1918                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1919                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1920                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1921                                 gen_nullptr_check(s1);
1922                                 gen_bound_check;
1923                         }
1924                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1925                         break;
1926
1927
1928                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1929
1930                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1931                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
1932
1933                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1934                                 disp = dseg_addaddress(cd, NULL);
1935                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1936
1937                                 /* must be calculated before codegen_addpatchref */
1938
1939                                 if (opt_showdisassemble)
1940                                         disp -= PATCHER_CALL_SIZE;
1941
1942 /*                              PROFILE_CYCLE_STOP; */
1943
1944                                 codegen_addpatchref(cd, PATCHER_get_putstatic, uf, disp);
1945
1946                                 if (opt_showdisassemble) {
1947                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
1948                                 }
1949
1950 /*                              PROFILE_CYCLE_START; */
1951                         }
1952                         else {
1953                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
1954
1955                                 fieldtype = fi->type;
1956
1957                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1958                                         PROFILE_CYCLE_STOP;
1959
1960                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
1961
1962                                         if (opt_showdisassemble) {
1963                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
1964                                         }
1965
1966                                         PROFILE_CYCLE_START;
1967                                 }
1968
1969                                 disp = dseg_addaddress(cd, &(fi->value));
1970                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1971                         }
1972
1973                         /* This approach is much faster than moving the field
1974                            address inline into a register. */
1975
1976                         M_ALD(REG_ITMP1, RIP, disp);
1977
1978                         switch (fieldtype) {
1979                         case TYPE_INT:
1980                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1981                                 M_ILD(d, REG_ITMP1, 0);
1982                                 break;
1983                         case TYPE_LNG:
1984                         case TYPE_ADR:
1985                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1986                                 M_LLD(d, REG_ITMP1, 0);
1987                                 break;
1988                         case TYPE_FLT:
1989                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1990                                 M_FLD(d, REG_ITMP1, 0);
1991                                 break;
1992                         case TYPE_DBL:
1993                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1994                                 M_DLD(d, REG_ITMP1, 0);
1995                                 break;
1996                         }
1997                         emit_store_dst(jd, iptr, d);
1998                         break;
1999
2000                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2001
2002                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2003                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2004
2005                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2006                                 disp = dseg_addaddress(cd, NULL);
2007                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2008
2009                                 /* must be calculated before codegen_addpatchref */
2010
2011                                 if (opt_showdisassemble)
2012                                         disp -= PATCHER_CALL_SIZE;
2013
2014 /*                              PROFILE_CYCLE_STOP; */
2015
2016                                 codegen_addpatchref(cd, PATCHER_get_putstatic, uf, disp);
2017
2018                                 if (opt_showdisassemble) {
2019                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2020                                 }
2021
2022 /*                              PROFILE_CYCLE_START; */
2023                         }
2024                         else {
2025                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2026
2027                                 fieldtype = fi->type;
2028
2029                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2030                                         PROFILE_CYCLE_STOP;
2031
2032                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2033
2034                                         if (opt_showdisassemble) {
2035                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2036                                                 disp -= PATCHER_CALL_SIZE;
2037                                         }
2038
2039                                         PROFILE_CYCLE_START;
2040                                 }
2041
2042                                 disp = dseg_addaddress(cd, &(fi->value));
2043                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2044                         }
2045
2046                         /* This approach is much faster than moving the field
2047                            address inline into a register. */
2048
2049                         M_ALD(REG_ITMP1, RIP, disp);
2050
2051                         switch (fieldtype) {
2052                         case TYPE_INT:
2053                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2054                                 M_IST(s1, REG_ITMP1, 0);
2055                                 break;
2056                         case TYPE_LNG:
2057                         case TYPE_ADR:
2058                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2059                                 M_LST(s1, REG_ITMP1, 0);
2060                                 break;
2061                         case TYPE_FLT:
2062                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
2063                                 M_FST(s1, REG_ITMP1, 0);
2064                                 break;
2065                         case TYPE_DBL:
2066                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
2067                                 M_DST(s1, REG_ITMP1, 0);
2068                                 break;
2069                         }
2070                         break;
2071
2072                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2073                                           /* val = value (in current instruction)     */
2074                                           /* following NOP)                           */
2075
2076                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2077                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2078
2079                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2080                                 disp = dseg_addaddress(cd, NULL);
2081                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2082
2083                                 /* must be calculated before codegen_addpatchref */
2084
2085                                 if (opt_showdisassemble)
2086                                         disp -= PATCHER_CALL_SIZE;
2087
2088
2089 /*                              PROFILE_CYCLE_STOP; */
2090
2091                                 codegen_addpatchref(cd, PATCHER_get_putstatic, uf, disp);
2092
2093                                 if (opt_showdisassemble) {
2094                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2095                                 }
2096
2097 /*                              PROFILE_CYCLE_START; */
2098                         }
2099                         else {
2100                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2101
2102                                 fieldtype = fi->type;
2103
2104                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2105                                         PROFILE_CYCLE_STOP;
2106
2107                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2108
2109                                         if (opt_showdisassemble) {
2110                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2111                                         }
2112
2113                                         PROFILE_CYCLE_START;
2114                                 }
2115
2116                                 disp = dseg_addaddress(cd, &(fi->value));
2117                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2118                         }
2119
2120                         /* This approach is much faster than moving the field
2121                            address inline into a register. */
2122
2123                         M_ALD(REG_ITMP1, RIP, disp);
2124
2125                         switch (fieldtype) {
2126                         case TYPE_INT:
2127                         case TYPE_FLT:
2128                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2129                                 break;
2130                         case TYPE_LNG:
2131                         case TYPE_ADR:
2132                         case TYPE_DBL:
2133                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
2134                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2135                                 else {
2136                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2137                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
2138                                 }
2139                                 break;
2140                         }
2141                         break;
2142
2143                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2144
2145                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2146                         gen_nullptr_check(s1);
2147
2148                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2149                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2150
2151 /*                              PROFILE_CYCLE_STOP; */
2152
2153                                 codegen_addpatchref(cd, PATCHER_get_putfield, uf, 0);
2154
2155                                 if (opt_showdisassemble) {
2156                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2157                                 }
2158
2159 /*                              PROFILE_CYCLE_START; */
2160
2161                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2162                                 disp = 0;
2163                         }
2164                         else {
2165                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2166
2167                                 fieldtype = fi->type;
2168                                 disp = fi->offset;
2169                         }
2170
2171                         switch (fieldtype) {
2172                         case TYPE_INT:
2173                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
2174                                 M_ILD32(d, s1, disp);
2175                                 break;
2176                         case TYPE_LNG:
2177                         case TYPE_ADR:
2178                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
2179                                 M_LLD32(d, s1, disp);
2180                                 break;
2181                         case TYPE_FLT:
2182                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2183                                 M_FLD32(d, s1, disp);
2184                                 break;
2185                         case TYPE_DBL:
2186                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2187                                 M_DLD32(d, s1, disp);
2188                                 break;
2189                         }
2190                         emit_store_dst(jd, iptr, d);
2191                         break;
2192
2193                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2194
2195                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2196                         gen_nullptr_check(s1);
2197
2198                         s2 = emit_load_s2(jd, iptr, REG_IFTMP);
2199
2200                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2201                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2202
2203 /*                              PROFILE_CYCLE_STOP; */
2204
2205                                 codegen_addpatchref(cd, PATCHER_get_putfield, uf, 0);
2206
2207                                 if (opt_showdisassemble) {
2208                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2209                                 }
2210
2211 /*                              PROFILE_CYCLE_START; */
2212
2213                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2214                                 disp = 0;
2215                         }
2216                         else {
2217                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2218
2219                                 fieldtype = fi->type;
2220                                 disp = fi->offset;
2221                         }
2222
2223                         switch (fieldtype) {
2224                         case TYPE_INT:
2225                                 M_IST32(s2, s1, disp);
2226                                 break;
2227                         case TYPE_LNG:
2228                         case TYPE_ADR:
2229                                 M_LST32(s2, s1, disp);
2230                                 break;
2231                         case TYPE_FLT:
2232                                 M_FST32(s2, s1, disp);
2233                                 break;
2234                         case TYPE_DBL:
2235                                 M_DST32(s2, s1, disp);
2236                                 break;
2237                         }
2238                         break;
2239
2240                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2241                                           /* val = value (in current instruction)     */
2242                                           /* following NOP)                           */
2243
2244                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2245                         gen_nullptr_check(s1);
2246
2247                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2248                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2249
2250 /*                              PROFILE_CYCLE_STOP; */
2251
2252                                 codegen_addpatchref(cd, PATCHER_putfieldconst, uf, 0);
2253
2254                                 if (opt_showdisassemble) {
2255                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2256                                 }
2257
2258 /*                              PROFILE_CYCLE_START; */
2259
2260                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2261                                 disp = 0;
2262                         }
2263                         else {
2264                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2265
2266                                 fieldtype = fi->type;
2267                                 disp = fi->offset;
2268                         }
2269
2270                         switch (fieldtype) {
2271                         case TYPE_INT:
2272                         case TYPE_FLT:
2273                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2274                                 break;
2275                         case TYPE_LNG:
2276                         case TYPE_ADR:
2277                         case TYPE_DBL:
2278                                 /* XXX why no check for IS_IMM32? */
2279                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2280                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2281                                 break;
2282                         }
2283                         break;
2284
2285
2286                 /* branch operations **************************************************/
2287
2288                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2289
2290                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2291                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2292
2293                         PROFILE_CYCLE_STOP;
2294
2295 #ifdef ENABLE_VERIFIER
2296                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2297                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2298                                                                         iptr->sx.s23.s2.uc, 0);
2299
2300                                 if (opt_showdisassemble) {
2301                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2302                                 }
2303                         }
2304 #endif /* ENABLE_VERIFIER */
2305
2306                         M_CALL_IMM(0);                            /* passing exception pc */
2307                         M_POP(REG_ITMP2_XPC);
2308
2309                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2310                         M_JMP(REG_ITMP3);
2311                         break;
2312
2313                 case ICMD_GOTO:         /* ... ==> ...                                */
2314
2315                         M_JMP_IMM(0);
2316                         codegen_addreference(cd, iptr->dst.block);
2317                         break;
2318
2319                 case ICMD_JSR:          /* ... ==> ...                                */
2320
2321                         M_CALL_IMM(0);
2322                         codegen_addreference(cd, iptr->sx.s23.s3.jsrtarget.block);
2323                         break;
2324
2325                 case ICMD_RET:          /* ... ==> ...                                */
2326                                         /* s1.localindex = local variable                       */
2327
2328                         M_JMP_IMM(0);
2329                         codegen_addreference(cd, iptr->dst.block);
2330                         break;
2331
2332                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2333
2334                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2335                         M_TEST(s1);
2336                         M_BEQ(0);
2337                         codegen_addreference(cd, iptr->dst.block);
2338                         break;
2339
2340                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2341
2342                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2343                         M_TEST(s1);
2344                         M_BNE(0);
2345                         codegen_addreference(cd, iptr->dst.block);
2346                         break;
2347
2348                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2349
2350                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2351                         M_ICMP_IMM(iptr->sx.val.i, s1);
2352                         M_BEQ(0);
2353                         codegen_addreference(cd, iptr->dst.block);
2354                         break;
2355
2356                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2357
2358                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2359                         M_ICMP_IMM(iptr->sx.val.i, s1);
2360                         M_BLT(0);
2361                         codegen_addreference(cd, iptr->dst.block);
2362                         break;
2363
2364                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2365
2366                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2367                         M_ICMP_IMM(iptr->sx.val.i, s1);
2368                         M_BLE(0);
2369                         codegen_addreference(cd, iptr->dst.block);
2370                         break;
2371
2372                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2373
2374                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2375                         M_ICMP_IMM(iptr->sx.val.i, s1);
2376                         M_BNE(0);
2377                         codegen_addreference(cd, iptr->dst.block);
2378                         break;
2379
2380                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2381
2382                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2383                         M_ICMP_IMM(iptr->sx.val.i, s1);
2384                         M_BGT(0);
2385                         codegen_addreference(cd, iptr->dst.block);
2386                         break;
2387
2388                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2389
2390                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2391                         M_ICMP_IMM(iptr->sx.val.i, s1);
2392                         M_BGE(0);
2393                         codegen_addreference(cd, iptr->dst.block);
2394                         break;
2395
2396                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2397
2398                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2399                         if (IS_IMM32(iptr->sx.val.l))
2400                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2401                         else {
2402                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2403                                 M_LCMP(REG_ITMP2, s1);
2404                         }
2405                         M_BEQ(0);
2406                         codegen_addreference(cd, iptr->dst.block);
2407                         break;
2408
2409                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2410
2411                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2412                         if (IS_IMM32(iptr->sx.val.l))
2413                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2414                         else {
2415                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2416                                 M_LCMP(REG_ITMP2, s1);
2417                         }
2418                         M_BLT(0);
2419                         codegen_addreference(cd, iptr->dst.block);
2420                         break;
2421
2422                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2423
2424                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2425                         if (IS_IMM32(iptr->sx.val.l))
2426                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2427                         else {
2428                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2429                                 M_LCMP(REG_ITMP2, s1);
2430                         }
2431                         M_BLE(0);
2432                         codegen_addreference(cd, iptr->dst.block);
2433                         break;
2434
2435                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2436
2437                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2438                         if (IS_IMM32(iptr->sx.val.l))
2439                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2440                         else {
2441                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2442                                 M_LCMP(REG_ITMP2, s1);
2443                         }
2444                         M_BNE(0);
2445                         codegen_addreference(cd, iptr->dst.block);
2446                         break;
2447
2448                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2449
2450                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2451                         if (IS_IMM32(iptr->sx.val.l))
2452                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2453                         else {
2454                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2455                                 M_LCMP(REG_ITMP2, s1);
2456                         }
2457                         M_BGT(0);
2458                         codegen_addreference(cd, iptr->dst.block);
2459                         break;
2460
2461                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2462
2463                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2464                         if (IS_IMM32(iptr->sx.val.l))
2465                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2466                         else {
2467                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2468                                 M_LCMP(REG_ITMP2, s1);
2469                         }
2470                         M_BGE(0);
2471                         codegen_addreference(cd, iptr->dst.block);
2472                         break;
2473
2474                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2475
2476                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2477                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2478                         M_ICMP(s2, s1);
2479                         M_BEQ(0);
2480                         codegen_addreference(cd, iptr->dst.block);
2481                         break;
2482
2483                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2484                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2485
2486                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2487                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2488                         M_LCMP(s2, s1);
2489                         M_BEQ(0);
2490                         codegen_addreference(cd, iptr->dst.block);
2491                         break;
2492
2493                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2494
2495                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2496                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2497                         M_ICMP(s2, s1);
2498                         M_BNE(0);
2499                         codegen_addreference(cd, iptr->dst.block);
2500                         break;
2501
2502                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2503                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2504
2505                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2506                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2507                         M_LCMP(s2, s1);
2508                         M_BNE(0);
2509                         codegen_addreference(cd, iptr->dst.block);
2510                         break;
2511
2512                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2513
2514                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2515                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2516                         M_ICMP(s2, s1);
2517                         M_BLT(0);
2518                         codegen_addreference(cd, iptr->dst.block);
2519                         break;
2520
2521                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2522
2523                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2524                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2525                         M_LCMP(s2, s1);
2526                         M_BLT(0);
2527                         codegen_addreference(cd, iptr->dst.block);
2528                         break;
2529
2530                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2531
2532                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2533                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2534                         M_ICMP(s2, s1);
2535                         M_BGT(0);
2536                         codegen_addreference(cd, iptr->dst.block);
2537                         break;
2538
2539                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2540
2541                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2542                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2543                         M_LCMP(s2, s1);
2544                         M_BGT(0);
2545                         codegen_addreference(cd, iptr->dst.block);
2546                         break;
2547
2548                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2549
2550                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2551                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2552                         M_ICMP(s2, s1);
2553                         M_BLE(0);
2554                         codegen_addreference(cd, iptr->dst.block);
2555                         break;
2556
2557                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2558
2559                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2560                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2561                         M_LCMP(s2, s1);
2562                         M_BLE(0);
2563                         codegen_addreference(cd, iptr->dst.block);
2564                         break;
2565
2566                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2567
2568                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2569                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2570                         M_ICMP(s2, s1);
2571                         M_BGE(0);
2572                         codegen_addreference(cd, iptr->dst.block);
2573                         break;
2574
2575                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2576
2577                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2578                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2579                         M_LCMP(s2, s1);
2580                         M_BGE(0);
2581                         codegen_addreference(cd, iptr->dst.block);
2582                         break;
2583
2584                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2585                 case ICMD_LRETURN:
2586
2587                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2588                         M_INTMOVE(s1, REG_RESULT);
2589                         goto nowperformreturn;
2590
2591                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2592
2593                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2594                         M_INTMOVE(s1, REG_RESULT);
2595
2596 #ifdef ENABLE_VERIFIER
2597                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2598                                 PROFILE_CYCLE_STOP;
2599
2600                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2601                                                                         iptr->sx.s23.s2.uc, 0);
2602
2603                                 if (opt_showdisassemble) {
2604                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2605                                 }
2606
2607                                 PROFILE_CYCLE_START;
2608                         }
2609 #endif /* ENABLE_VERIFIER */
2610                         goto nowperformreturn;
2611
2612                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2613                 case ICMD_DRETURN:
2614
2615                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2616                         M_FLTMOVE(s1, REG_FRESULT);
2617                         goto nowperformreturn;
2618
2619                 case ICMD_RETURN:      /* ...  ==> ...                                */
2620
2621 nowperformreturn:
2622                         {
2623                         s4 i, p;
2624
2625                         p = cd->stackframesize;
2626
2627 #if !defined(NDEBUG)
2628                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2629                                 emit_verbosecall_exit(jd);
2630 #endif /* !defined(NDEBUG) */
2631
2632 #if defined(ENABLE_THREADS)
2633                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2634                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2635
2636                                 /* we need to save the proper return value */
2637                                 switch (iptr->opc) {
2638                                 case ICMD_IRETURN:
2639                                 case ICMD_ARETURN:
2640                                 case ICMD_LRETURN:
2641                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2642                                         break;
2643                                 case ICMD_FRETURN:
2644                                 case ICMD_DRETURN:
2645                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2646                                         break;
2647                                 }
2648
2649                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2650                                 M_CALL(REG_ITMP1);
2651
2652                                 /* and now restore the proper return value */
2653                                 switch (iptr->opc) {
2654                                 case ICMD_IRETURN:
2655                                 case ICMD_ARETURN:
2656                                 case ICMD_LRETURN:
2657                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2658                                         break;
2659                                 case ICMD_FRETURN:
2660                                 case ICMD_DRETURN:
2661                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2662                                         break;
2663                                 }
2664                         }
2665 #endif
2666
2667                         /* restore saved registers */
2668
2669                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2670                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2671                         }
2672                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2673                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2674                         }
2675
2676                         /* deallocate stack */
2677
2678                         if (cd->stackframesize)
2679                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2680
2681                         /* generate method profiling code */
2682
2683                         PROFILE_CYCLE_STOP;
2684
2685                         M_RET;
2686                         }
2687                         break;
2688
2689
2690                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2691                         {
2692                                 s4 i, l;
2693                                 branch_target_t *table;
2694
2695                                 table = iptr->dst.table;
2696
2697                                 l = iptr->sx.s23.s2.tablelow;
2698                                 i = iptr->sx.s23.s3.tablehigh;
2699
2700                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2701                                 M_INTMOVE(s1, REG_ITMP1);
2702
2703                                 if (l != 0)
2704                                         M_ISUB_IMM(l, REG_ITMP1);
2705
2706                                 /* number of targets */
2707                                 i = i - l + 1;
2708
2709                 /* range check */
2710                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2711                                 M_BA(0);
2712
2713                                 codegen_addreference(cd, table[0].block); /* default target */
2714
2715                                 /* build jump table top down and use address of lowest entry */
2716
2717                                 table += i;
2718
2719                                 while (--i >= 0) {
2720                                         dseg_addtarget(cd, table->block);
2721                                         --table;
2722                                 }
2723
2724                                 /* length of dataseg after last dseg_addtarget is used
2725                                    by load */
2726
2727                                 M_MOV_IMM(0, REG_ITMP2);
2728                                 dseg_adddata(cd);
2729                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2730                                 M_JMP(REG_ITMP1);
2731                         }
2732                         break;
2733
2734
2735                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2736                         {
2737                                 s4 i;
2738                                 lookup_target_t *lookup;
2739
2740                                 lookup = iptr->dst.lookup;
2741
2742                                 i = iptr->sx.s23.s2.lookupcount;
2743
2744                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2745                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2746
2747                                 while (--i >= 0) {
2748                                         M_ICMP_IMM(lookup->value, s1);
2749                                         M_BEQ(0);
2750                                         codegen_addreference(cd, lookup->target.block);
2751                                         lookup++;
2752                                 }
2753
2754                                 M_JMP_IMM(0);
2755
2756                                 codegen_addreference(cd, iptr->sx.s23.s3.lookupdefault.block);
2757                         }
2758                         break;
2759
2760
2761                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2762
2763                         bte = iptr->sx.s23.s3.bte;
2764                         md  = bte->md;
2765                         goto gen_method;
2766
2767                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2768
2769                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2770                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2771                 case ICMD_INVOKEINTERFACE:
2772
2773                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2774                                 lm = NULL;
2775                                 um = iptr->sx.s23.s3.um;
2776                                 md = um->methodref->parseddesc.md;
2777                         }
2778                         else {
2779                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2780                                 um = NULL;
2781                                 md = lm->parseddesc;
2782                         }
2783
2784 gen_method:
2785                         s3 = md->paramcount;
2786
2787                         MCODECHECK((20 * s3) + 128);
2788
2789                         /* copy arguments to registers or stack location */
2790
2791                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2792                                 s1 = iptr->sx.s23.s2.args[s3];
2793                                 var1 = jd->var + s1;
2794
2795                                 /* Already Preallocated (ARGVAR) ? */
2796                                 if (var1->flags & PREALLOC)
2797                                         continue;
2798
2799                                 if (IS_INT_LNG_TYPE(var1->type)) {
2800                                         if (!md->params[s3].inmemory) {
2801                                                 s1 = rd->argintregs[md->params[s3].regoff];
2802                                                 d = emit_load(jd, iptr, var1, s1);
2803                                                 M_INTMOVE(d, s1);
2804                                         }
2805                                         else {
2806                                                 d = emit_load(jd, iptr, var1, REG_ITMP1);
2807                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
2808                                         }
2809                                 }
2810                                 else {
2811                                         if (!md->params[s3].inmemory) {
2812                                                 s1 = rd->argfltregs[md->params[s3].regoff];
2813                                                 d = emit_load(jd, iptr, var1, s1);
2814                                                 M_FLTMOVE(d, s1);
2815                                         }
2816                                         else {
2817                                                 d = emit_load(jd, iptr, var1, REG_FTMP1);
2818
2819                                                 if (IS_2_WORD_TYPE(var1->type))
2820                                                         M_DST(d, REG_SP, md->params[s3].regoff * 8);
2821                                                 else
2822                                                         M_FST(d, REG_SP, md->params[s3].regoff * 8);
2823                                         }
2824                                 }
2825                         }
2826
2827                         /* generate method profiling code */
2828
2829                         PROFILE_CYCLE_STOP;
2830
2831                         switch (iptr->opc) {
2832                         case ICMD_BUILTIN:
2833                                 M_MOV_IMM(bte->fp, REG_ITMP1);
2834                                 M_CALL(REG_ITMP1);
2835
2836                                 if (INSTRUCTION_MUST_CHECK(iptr)) {
2837                                         M_TEST(REG_RESULT);
2838                                         M_BEQ(0);
2839                                         codegen_add_fillinstacktrace_ref(cd);
2840                                 }
2841                                 break;
2842
2843                         case ICMD_INVOKESPECIAL:
2844                                 M_TEST(REG_A0);
2845                                 M_BEQ(0);
2846                                 codegen_add_nullpointerexception_ref(cd);
2847
2848                                 /* fall through */
2849
2850                         case ICMD_INVOKESTATIC:
2851                                 if (lm == NULL) {
2852                                         disp = dseg_addaddress(cd, NULL);
2853                                         disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2854
2855                                         /* must be calculated before codegen_addpatchref */
2856
2857                                         if (opt_showdisassemble)
2858                                                 disp -= PATCHER_CALL_SIZE;
2859
2860                                         codegen_addpatchref(cd, PATCHER_invokestatic_special,
2861                                                                                 um, disp);
2862
2863                                         if (opt_showdisassemble) {
2864                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2865                                         }
2866
2867 /*                                      a = 0; */
2868                                 }
2869                                 else {
2870                                         disp = dseg_addaddress(cd, lm->stubroutine);
2871                                         disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2872
2873 /*                                      a = (ptrint) lm->stubroutine; */
2874                                 }
2875
2876 /*                              M_MOV_IMM(a, REG_ITMP2); */
2877                                 M_ALD(REG_ITMP2, RIP, disp);
2878                                 M_CALL(REG_ITMP2);
2879                                 break;
2880
2881                         case ICMD_INVOKEVIRTUAL:
2882                                 gen_nullptr_check(REG_A0);
2883
2884                                 if (lm == NULL) {
2885                                         codegen_addpatchref(cd, PATCHER_invokevirtual, um, 0);
2886
2887                                         if (opt_showdisassemble) {
2888                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2889                                         }
2890
2891                                         s1 = 0;
2892                                 }
2893                                 else
2894                                         s1 = OFFSET(vftbl_t, table[0]) +
2895                                                 sizeof(methodptr) * lm->vftblindex;
2896
2897                                 M_ALD(REG_METHODPTR, REG_A0,
2898                                           OFFSET(java_objectheader, vftbl));
2899                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2900                                 M_CALL(REG_ITMP3);
2901                                 break;
2902
2903                         case ICMD_INVOKEINTERFACE:
2904                                 gen_nullptr_check(REG_A0);
2905
2906                                 if (lm == NULL) {
2907                                         codegen_addpatchref(cd, PATCHER_invokeinterface, um, 0);
2908
2909                                         if (opt_showdisassemble) {
2910                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2911                                         }
2912
2913                                         s1 = 0;
2914                                         s2 = 0;
2915                                 }
2916                                 else {
2917                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2918                                                 sizeof(methodptr) * lm->class->index;
2919
2920                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2921                                 }
2922
2923                                 M_ALD(REG_METHODPTR, REG_A0,
2924                                           OFFSET(java_objectheader, vftbl));
2925                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2926                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2927                                 M_CALL(REG_ITMP3);
2928                                 break;
2929                         }
2930
2931                         /* generate method profiling code */
2932
2933                         PROFILE_CYCLE_START;
2934
2935                         /* store return value */
2936
2937                         d = md->returntype.type;
2938
2939                         if (d != TYPE_VOID) {
2940                                 if (IS_INT_LNG_TYPE(d)) {
2941                                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2942                                         M_INTMOVE(REG_RESULT, s1);
2943                                 }
2944                                 else {
2945                                         s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2946                                         M_FLTMOVE(REG_FRESULT, s1);
2947                                 }
2948                                 emit_store_dst(jd, iptr, s1);
2949                         }
2950                         break;
2951
2952
2953                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2954
2955                                       /* val.a: (classinfo *) superclass              */
2956
2957                         /*  superclass is an interface:
2958                          *
2959                          *  OK if ((sub == NULL) ||
2960                          *         (sub->vftbl->interfacetablelength > super->index) &&
2961                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
2962                          *
2963                          *  superclass is a class:
2964                          *
2965                          *  OK if ((sub == NULL) || (0
2966                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
2967                          *         super->vftbl->diffval));
2968                          */
2969
2970                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2971                                 /* object type cast-check */
2972
2973                                 classinfo *super;
2974                                 vftbl_t   *supervftbl;
2975                                 s4         superindex;
2976
2977                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2978                                         super = NULL;
2979                                         superindex = 0;
2980                                         supervftbl = NULL;
2981                                 }
2982                                 else {
2983                                         super = iptr->sx.s23.s3.c.cls;
2984                                         superindex = super->index;
2985                                         supervftbl = super->vftbl;
2986                                 }
2987
2988 #if defined(ENABLE_THREADS)
2989                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
2990 #endif
2991                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2992
2993                                 /* calculate interface checkcast code size */
2994
2995                                 s2 = 3; /* mov_membase_reg */
2996                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
2997
2998                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
2999                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3000                                         3 /* test */ + 6 /* jcc */;
3001
3002                                 if (super == NULL)
3003                                         s2 += (opt_showdisassemble ? 5 : 0);
3004
3005                                 /* calculate class checkcast code size */
3006
3007                                 s3 = 3; /* mov_membase_reg */
3008                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3009                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3010
3011 #if 0
3012                                 if (s1 != REG_ITMP1) {
3013                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3014                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3015                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3016                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3017                                         a += 3;    /* sub */
3018
3019                                 } else
3020 #endif
3021                                         {
3022                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3023                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3024                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3025                                         }
3026
3027                                 s3 += 3 /* cmp */ + 6 /* jcc */;
3028
3029                                 if (super == NULL)
3030                                         s3 += (opt_showdisassemble ? 5 : 0);
3031
3032                                 /* if class is not resolved, check which code to call */
3033
3034                                 if (super == NULL) {
3035                                         M_TEST(s1);
3036                                         M_BEQ(6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3037
3038                                         codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3039                                                                                 iptr->sx.s23.s3.c.ref, 0);
3040
3041                                         if (opt_showdisassemble) {
3042                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3043                                         }
3044
3045                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
3046                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
3047                                         M_BEQ(s2 + 5);
3048                                 }
3049
3050                                 /* interface checkcast code */
3051
3052                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
3053                                         if (super != NULL) {
3054                                                 M_TEST(s1);
3055                                                 M_BEQ(s2);
3056                                         }
3057
3058                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3059
3060                                         if (super == NULL) {
3061                                                 codegen_addpatchref(cd,
3062                                                                                         PATCHER_checkcast_instanceof_interface,
3063                                                                                         iptr->sx.s23.s3.c.ref,
3064                                                                                         0);
3065
3066                                                 if (opt_showdisassemble) {
3067                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3068                                                 }
3069                                         }
3070
3071                                         emit_movl_membase32_reg(cd, REG_ITMP2,
3072                                                                                           OFFSET(vftbl_t, interfacetablelength),
3073                                                                                           REG_ITMP3);
3074                                         /* XXX TWISTI: should this be int arithmetic? */
3075                                         M_LSUB_IMM32(superindex, REG_ITMP3);
3076                                         M_TEST(REG_ITMP3);
3077                                         M_BLE(0);
3078                                         codegen_add_classcastexception_ref(cd, s1);
3079                                         emit_mov_membase32_reg(cd, REG_ITMP2,
3080                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
3081                                                                                          superindex * sizeof(methodptr*),
3082                                                                                          REG_ITMP3);
3083                                         M_TEST(REG_ITMP3);
3084                                         M_BEQ(0);
3085                                         codegen_add_classcastexception_ref(cd, s1);
3086
3087                                         if (super == NULL)
3088                                                 M_JMP_IMM(s3);
3089                                 }
3090
3091                                 /* class checkcast code */
3092
3093                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
3094                                         if (super != NULL) {
3095                                                 M_TEST(s1);
3096                                                 M_BEQ(s3);
3097                                         }
3098
3099                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3100
3101                                         if (super == NULL) {
3102                                                 codegen_addpatchref(cd, PATCHER_checkcast_class,
3103                                                                                         iptr->sx.s23.s3.c.ref,
3104                                                                                         0);
3105
3106                                                 if (opt_showdisassemble) {
3107                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3108                                                 }
3109                                         }
3110
3111                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3112 #if defined(ENABLE_THREADS)
3113                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3114 #endif
3115                                         emit_movl_membase32_reg(cd, REG_ITMP2,
3116                                                                                           OFFSET(vftbl_t, baseval),
3117                                                                                           REG_ITMP2);
3118                                         /*                                      if (s1 != REG_ITMP1) { */
3119                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3120                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
3121                                         /*                                                                                              REG_ITMP1); */
3122                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3123                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3124                                         /*                                                                                              REG_ITMP3); */
3125                                         /*  #if defined(ENABLE_THREADS) */
3126                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3127                                         /*  #endif */
3128                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
3129
3130                                         /*                                      } else { */
3131                                         emit_movl_membase32_reg(cd, REG_ITMP3,
3132                                                                                           OFFSET(vftbl_t, baseval),
3133                                                                                           REG_ITMP3);
3134                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3135                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3136                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3137                                         /*                                      } */
3138 #if defined(ENABLE_THREADS)
3139                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3140 #endif
3141                                         M_LCMP(REG_ITMP3, REG_ITMP2);
3142                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3143                                         codegen_add_classcastexception_ref(cd, s1);
3144                                 }
3145
3146                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
3147                         }
3148                         else {
3149                                 /* array type cast-check */
3150
3151                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
3152                                 M_INTMOVE(s1, REG_A0);
3153
3154                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3155                                         codegen_addpatchref(cd, PATCHER_builtin_arraycheckcast,
3156                                                                                 iptr->sx.s23.s3.c.ref, 0);
3157
3158                                         if (opt_showdisassemble) {
3159                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3160                                         }
3161                                 }
3162
3163                                 M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
3164                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
3165                                 M_CALL(REG_ITMP1);
3166
3167                                 /* s1 may have been destroyed over the function call */
3168                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
3169                                 M_TEST(REG_RESULT);
3170                                 M_BEQ(0);
3171                                 codegen_add_classcastexception_ref(cd, s1);
3172
3173                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
3174                         }
3175
3176                         M_INTMOVE(s1, d);
3177                         emit_store_dst(jd, iptr, d);
3178                         break;
3179
3180                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3181
3182                                       /* val.a: (classinfo *) superclass              */
3183
3184                         /*  superclass is an interface:
3185                          *
3186                          *  return (sub != NULL) &&
3187                          *         (sub->vftbl->interfacetablelength > super->index) &&
3188                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3189                          *
3190                          *  superclass is a class:
3191                          *
3192                          *  return ((sub != NULL) && (0
3193                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3194                          *          super->vftbl->diffvall));
3195                          */
3196
3197                         {
3198                         classinfo *super;
3199                         vftbl_t   *supervftbl;
3200                         s4         superindex;
3201
3202                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3203                                 super = NULL;
3204                                 superindex = 0;
3205                                 supervftbl = NULL;
3206
3207                         } else {
3208                                 super = iptr->sx.s23.s3.c.cls;
3209                                 superindex = super->index;
3210                                 supervftbl = super->vftbl;
3211                         }
3212
3213 #if defined(ENABLE_THREADS)
3214             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3215 #endif
3216
3217                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
3218                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
3219                         if (s1 == d) {
3220                                 M_INTMOVE(s1, REG_ITMP1);
3221                                 s1 = REG_ITMP1;
3222                         }
3223
3224                         /* calculate interface instanceof code size */
3225
3226                         s2 = 3; /* mov_membase_reg */
3227                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3228                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3229                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3230                                 3 /* test */ + 4 /* setcc */;
3231
3232                         if (!super)
3233                                 s2 += (opt_showdisassemble ? 5 : 0);
3234
3235                         /* calculate class instanceof code size */
3236
3237                         s3 = 3; /* mov_membase_reg */
3238                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3239                         s3 += 10; /* mov_imm_reg */
3240                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3241                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3242                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3243                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3244                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3245                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3246                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3247
3248                         if (!super)
3249                                 s3 += (opt_showdisassemble ? 5 : 0);
3250
3251                         emit_alu_reg_reg(cd, ALU_XOR, d, d);
3252
3253                         /* if class is not resolved, check which code to call */
3254
3255                         if (!super) {
3256                                 emit_test_reg_reg(cd, s1, s1);
3257                                 emit_jcc(cd, CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3258                                                                                          7 + 6 + s2 + 5 + s3));
3259
3260                                 codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3261                                                                         iptr->sx.s23.s3.c.ref, 0);
3262
3263                                 if (opt_showdisassemble) {
3264                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3265                                 }
3266
3267                                 emit_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3268                                 emit_alul_imm_reg(cd, ALU_AND, ACC_INTERFACE, REG_ITMP3);
3269                                 emit_jcc(cd, CC_Z, s2 + 5);
3270                         }
3271
3272                         /* interface instanceof code */
3273
3274                         if (!super || (super->flags & ACC_INTERFACE)) {
3275                                 if (super) {
3276                                         emit_test_reg_reg(cd, s1, s1);
3277                                         emit_jcc(cd, CC_Z, s2);
3278                                 }
3279
3280                                 emit_mov_membase_reg(cd, s1,
3281                                                                            OFFSET(java_objectheader, vftbl),
3282                                                                            REG_ITMP1);
3283                                 if (!super) {
3284                                         codegen_addpatchref(cd,
3285                                                                                 PATCHER_checkcast_instanceof_interface,
3286                                                                                 iptr->sx.s23.s3.c.ref, 0);
3287
3288                                         if (opt_showdisassemble) {
3289                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3290                                         }
3291                                 }
3292
3293                                 emit_movl_membase32_reg(cd, REG_ITMP1,
3294                                                                                   OFFSET(vftbl_t, interfacetablelength),
3295                                                                                   REG_ITMP3);
3296                                 emit_alu_imm32_reg(cd, ALU_SUB, superindex, REG_ITMP3);
3297                                 emit_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3298
3299                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3300
3301                                 emit_jcc(cd, CC_LE, a);
3302                                 emit_mov_membase32_reg(cd, REG_ITMP1,
3303                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3304                                                                                  superindex * sizeof(methodptr*),
3305                                                                                  REG_ITMP1);
3306                                 emit_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3307                                 emit_setcc_reg(cd, CC_NE, d);
3308
3309                                 if (!super)
3310                                         emit_jmp_imm(cd, s3);
3311                         }
3312
3313                         /* class instanceof code */
3314
3315                         if (!super || !(super->flags & ACC_INTERFACE)) {
3316                                 if (super) {
3317                                         emit_test_reg_reg(cd, s1, s1);
3318                                         emit_jcc(cd, CC_E, s3);
3319                                 }
3320
3321                                 emit_mov_membase_reg(cd, s1,
3322                                                                            OFFSET(java_objectheader, vftbl),
3323                                                                            REG_ITMP1);
3324
3325                                 if (!super) {
3326                                         codegen_addpatchref(cd, PATCHER_instanceof_class,
3327                                                                                 iptr->sx.s23.s3.c.ref, 0);
3328
3329                                         if (opt_showdisassemble) {
3330                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3331                                         }
3332                                 }
3333
3334                                 emit_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3335 #if defined(ENABLE_THREADS)
3336                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3337 #endif
3338                                 emit_movl_membase_reg(cd, REG_ITMP1,
3339                                                                                 OFFSET(vftbl_t, baseval),
3340                                                                                 REG_ITMP1);
3341                                 emit_movl_membase_reg(cd, REG_ITMP2,
3342                                                                                 OFFSET(vftbl_t, diffval),
3343                                                                                 REG_ITMP3);
3344                                 emit_movl_membase_reg(cd, REG_ITMP2,
3345                                                                                 OFFSET(vftbl_t, baseval),
3346                                                                                 REG_ITMP2);
3347 #if defined(ENABLE_THREADS)
3348                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3349 #endif
3350                                 emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
3351                                 emit_alu_reg_reg(cd, ALU_XOR, d, d); /* may be REG_ITMP2 */
3352                                 emit_alu_reg_reg(cd, ALU_CMP, REG_ITMP3, REG_ITMP1);
3353                                 emit_setcc_reg(cd, CC_BE, d);
3354                         }
3355                         emit_store_dst(jd, iptr, d);
3356                         }
3357                         break;
3358
3359                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3360
3361                         /* check for negative sizes and copy sizes to stack if necessary  */
3362
3363                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
3364
3365                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
3366
3367                                 /* copy SAVEDVAR sizes to stack */
3368                                 s3 = iptr->sx.s23.s2.args[s1];
3369                                 var1 = jd->var + s3;
3370
3371                                 /* Already Preallocated (ARGVAR) ? */
3372                                 if (!(var1->flags & PREALLOC)) {
3373                                         s2 = emit_load(jd, iptr, var1, REG_ITMP1);
3374                                         M_LST(s2, REG_SP, s1 * 8);
3375                                 }
3376                         }
3377
3378                         /* is a patcher function set? */
3379
3380                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3381                                 codegen_addpatchref(cd, PATCHER_builtin_multianewarray,
3382                                                                         iptr->sx.s23.s3.c.ref, 0);
3383
3384                                 if (opt_showdisassemble) {
3385                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3386                                 }
3387                         }
3388
3389                         /* a0 = dimension count */
3390
3391                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
3392
3393                         /* a1 = classinfo */
3394
3395                         M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
3396
3397                         /* a2 = pointer to dimensions = stack pointer */
3398
3399                         M_MOV(REG_SP, REG_A2);
3400
3401                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3402                         M_CALL(REG_ITMP1);
3403
3404                         /* check for exception before result assignment */
3405
3406                         M_TEST(REG_RESULT);
3407                         M_BEQ(0);
3408                         codegen_add_fillinstacktrace_ref(cd);
3409
3410                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
3411                         M_INTMOVE(REG_RESULT, s1);
3412                         emit_store_dst(jd, iptr, s1);
3413                         break;
3414
3415                 default:
3416                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3417                         return false;
3418         } /* switch */
3419
3420         } /* for instruction */
3421
3422         /* copy values to interface registers */
3423
3424         len = bptr->outdepth;
3425         MCODECHECK(512);
3426
3427         /* At the end of a basic block we may have to append some nops,
3428            because the patcher stub calling code might be longer than the
3429            actual instruction. So codepatching does not change the
3430            following block unintentionally. */
3431
3432         if (cd->mcodeptr < cd->lastmcodeptr) {
3433                 while (cd->mcodeptr < cd->lastmcodeptr) {
3434                         M_NOP;
3435                 }
3436         }
3437
3438         } /* if (bptr -> flags >= BBREACHED) */
3439         } /* for basic block */
3440
3441         dseg_createlinenumbertable(cd);
3442
3443         /* generate stubs */
3444
3445         emit_exception_stubs(jd);
3446         emit_patcher_stubs(jd);
3447 #if 0
3448         emit_replacement_stubs(jd);
3449 #endif
3450
3451         codegen_finish(jd);
3452
3453         /* everything's ok */
3454
3455         return true;
3456 }
3457
3458
3459 /* createcompilerstub **********************************************************
3460
3461    Creates a stub routine which calls the compiler.
3462
3463 *******************************************************************************/
3464
3465 #define COMPILERSTUB_DATASIZE    3 * SIZEOF_VOID_P
3466 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
3467
3468 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
3469
3470
3471 u1 *createcompilerstub(methodinfo *m)
3472 {
3473         u1          *s;                     /* memory to hold the stub            */
3474         ptrint      *d;
3475         codeinfo    *code;
3476         codegendata *cd;
3477         s4           dumpsize;
3478
3479         s = CNEW(u1, COMPILERSTUB_SIZE);
3480
3481         /* set data pointer and code pointer */
3482
3483         d = (ptrint *) s;
3484         s = s + COMPILERSTUB_DATASIZE;
3485
3486         /* mark start of dump memory area */
3487
3488         dumpsize = dump_size();
3489
3490         cd = DNEW(codegendata);
3491         cd->mcodeptr = s;
3492
3493         /* Store the codeinfo pointer in the same place as in the
3494            methodheader for compiled methods. */
3495
3496         code = code_codeinfo_new(m);
3497
3498         d[0] = (ptrint) asm_call_jit_compiler;
3499         d[1] = (ptrint) m;
3500         d[2] = (ptrint) code;
3501
3502         /* code for the stub */
3503
3504         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
3505         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
3506         M_JMP(REG_ITMP3);
3507
3508 #if defined(ENABLE_STATISTICS)
3509         if (opt_stat)
3510                 count_cstub_len += COMPILERSTUB_SIZE;
3511 #endif
3512
3513         /* release dump area */
3514
3515         dump_release(dumpsize);
3516
3517         return s;
3518 }
3519
3520
3521 /* createnativestub ************************************************************
3522
3523    Creates a stub routine which calls a native method.
3524
3525 *******************************************************************************/
3526
3527 u1 *createnativestub(functionptr f, jitdata *jd, methoddesc *nmd)
3528 {
3529         methodinfo   *m;
3530         codeinfo     *code;
3531         codegendata  *cd;
3532         registerdata *rd;
3533         methoddesc   *md;
3534         s4            nativeparams;
3535         s4            i, j;                 /* count variables                    */
3536         s4            t;
3537         s4            s1, s2;
3538
3539         /* get required compiler data */
3540
3541         m    = jd->m;
3542         code = jd->code;
3543         cd   = jd->cd;
3544         rd   = jd->rd;
3545
3546         /* initialize variables */
3547
3548         md = m->parseddesc;
3549         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
3550
3551         /* calculate stack frame size */
3552
3553         cd->stackframesize =
3554                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
3555                 sizeof(localref_table) / SIZEOF_VOID_P +
3556                 INT_ARG_CNT + FLT_ARG_CNT +
3557                 1 +                       /* functionptr, TODO: store in data segment */
3558                 nmd->memuse;
3559
3560         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
3561
3562         /* create method header */
3563
3564         (void) dseg_addaddress(cd, code);                      /* CodeinfoPointer */
3565         (void) dseg_adds4(cd, cd->stackframesize * 8);         /* FrameSize       */
3566         (void) dseg_adds4(cd, 0);                              /* IsSync          */
3567         (void) dseg_adds4(cd, 0);                              /* IsLeaf          */
3568         (void) dseg_adds4(cd, 0);                              /* IntSave         */
3569         (void) dseg_adds4(cd, 0);                              /* FltSave         */
3570         (void) dseg_addlinenumbertablesize(cd);
3571         (void) dseg_adds4(cd, 0);                              /* ExTableSize     */
3572
3573         /* generate native method profiling code */
3574
3575         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3576                 /* count frequency */
3577
3578                 M_MOV_IMM(code, REG_ITMP3);
3579                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3580         }
3581
3582         /* generate stub code */
3583
3584         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3585
3586 #if !defined(NDEBUG)
3587         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3588                 emit_verbosecall_enter(jd);
3589 #endif
3590
3591         /* get function address (this must happen before the stackframeinfo) */
3592
3593 #if !defined(WITH_STATIC_CLASSPATH)
3594         if (f == NULL) {
3595                 codegen_addpatchref(cd, PATCHER_resolve_native, m, 0);
3596
3597                 if (opt_showdisassemble) {
3598                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3599                 }
3600         }
3601 #endif
3602
3603         M_MOV_IMM(f, REG_ITMP3);
3604
3605
3606         /* save integer and float argument registers */
3607
3608         for (i = 0, j = 0; i < md->paramcount; i++) {
3609                 if (!md->params[i].inmemory) {
3610                         s1 = md->params[i].regoff;
3611
3612                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3613                                 M_LST(rd->argintregs[s1], REG_SP, j * 8);
3614                         else
3615                                 M_DST(rd->argfltregs[s1], REG_SP, j * 8);
3616
3617                         j++;
3618                 }
3619         }
3620
3621         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3622
3623         /* create dynamic stack info */
3624
3625         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3626         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3627         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3628         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3629         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3630         M_CALL(REG_ITMP1);
3631
3632         /* restore integer and float argument registers */
3633
3634         for (i = 0, j = 0; i < md->paramcount; i++) {
3635                 if (!md->params[i].inmemory) {
3636                         s1 = md->params[i].regoff;
3637
3638                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3639                                 M_LLD(rd->argintregs[s1], REG_SP, j * 8);
3640                         else
3641                                 M_DLD(rd->argfltregs[s1], REG_SP, j * 8);
3642
3643                         j++;
3644                 }
3645         }
3646
3647         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3648
3649
3650         /* copy or spill arguments to new locations */
3651
3652         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3653                 t = md->paramtypes[i].type;
3654
3655                 if (IS_INT_LNG_TYPE(t)) {
3656                         if (!md->params[i].inmemory) {
3657                                 s1 = rd->argintregs[md->params[i].regoff];
3658
3659                                 if (!nmd->params[j].inmemory) {
3660                                         s2 = rd->argintregs[nmd->params[j].regoff];
3661                                         M_INTMOVE(s1, s2);
3662
3663                                 } else {
3664                                         s2 = nmd->params[j].regoff;
3665                                         M_LST(s1, REG_SP, s2 * 8);
3666                                 }
3667
3668                         } else {
3669                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3670                                 s2 = nmd->params[j].regoff;
3671                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
3672                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
3673                         }
3674
3675                 } else {
3676                         /* We only copy spilled float arguments, as the float argument    */
3677                         /* registers keep unchanged.                                      */
3678
3679                         if (md->params[i].inmemory) {
3680                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3681                                 s2 = nmd->params[j].regoff;
3682
3683                                 if (IS_2_WORD_TYPE(t)) {
3684                                         M_DLD(REG_FTMP1, REG_SP, s1 * 8);
3685                                         M_DST(REG_FTMP1, REG_SP, s2 * 8);
3686                                 } else {
3687                                         M_FLD(REG_FTMP1, REG_SP, s1 * 8);
3688                                         M_FST(REG_FTMP1, REG_SP, s2 * 8);
3689                                 }
3690                         }
3691                 }
3692         }
3693
3694         /* put class into second argument register */
3695
3696         if (m->flags & ACC_STATIC)
3697                 M_MOV_IMM(m->class, REG_A1);
3698
3699         /* put env into first argument register */
3700
3701         M_MOV_IMM(_Jv_env, REG_A0);
3702
3703         /* do the native function call */
3704
3705         M_CALL(REG_ITMP3);
3706
3707         /* save return value */
3708
3709         if (md->returntype.type != TYPE_VOID) {
3710                 if (IS_INT_LNG_TYPE(md->returntype.type))
3711                         M_LST(REG_RESULT, REG_SP, 0 * 8);
3712                 else
3713                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
3714         }
3715
3716 #if !defined(NDEBUG)
3717         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3718                 emit_verbosecall_exit(jd);
3719 #endif
3720
3721         /* remove native stackframe info */
3722
3723         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3724         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3725         M_CALL(REG_ITMP1);
3726         M_MOV(REG_RESULT, REG_ITMP3);
3727
3728         /* restore return value */
3729
3730         if (md->returntype.type != TYPE_VOID) {
3731                 if (IS_INT_LNG_TYPE(md->returntype.type))
3732                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
3733                 else
3734                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3735         }
3736
3737         /* remove stackframe */
3738
3739         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3740
3741         /* test for exception */
3742
3743         M_TEST(REG_ITMP3);
3744         M_BNE(1);
3745         M_RET;
3746
3747         /* handle exception */
3748
3749         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3750         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3751         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3752
3753         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3754         M_JMP(REG_ITMP3);
3755
3756         /* generate patcher stubs */
3757
3758         emit_patcher_stubs(jd);
3759
3760         codegen_finish(jd);
3761
3762         return code->entrypoint;
3763 }
3764
3765
3766 /*
3767  * These are local overrides for various environment variables in Emacs.
3768  * Please do not remove this and leave it at the end of the file, where
3769  * Emacs will automagically detect them.
3770  * ---------------------------------------------------------------------
3771  * Local variables:
3772  * mode: c
3773  * indent-tabs-mode: t
3774  * c-basic-offset: 4
3775  * tab-width: 4
3776  * End:
3777  * vim:noexpandtab:sw=4:ts=4:
3778  */