src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31             Edwin Steiner
  32
  33    $Id: codegen.c 5587 2006-09-30 10:16:48Z christian $
  34
  35 */
  36
  37
  38 #include "config.h"
  39
  40 #include <assert.h>
  41 #include <stdio.h>
  42
  43 #include "vm/types.h"
  44
  45 #include "md.h"
  46 #include "md-abi.h"
  47
  48 #include "vm/jit/x86_64/arch.h"
  49 #include "vm/jit/x86_64/codegen.h"
  50 #include "vm/jit/x86_64/emit.h"
  51
  52 #include "mm/memory.h"
  53 #include "native/jni.h"
  54 #include "native/native.h"
  55
  56 #if defined(ENABLE_THREADS)
  57 # include "threads/native/lock.h"
  58 #endif
  59
  60 #include "vm/builtin.h"
  61 #include "vm/exceptions.h"
  62 #include "vm/global.h"
  63 #include "vm/loader.h"
  64 #include "vm/options.h"
  65 #include "vm/statistics.h"
  66 #include "vm/stringlocal.h"
  67 #include "vm/vm.h"
  68 #include "vm/jit/asmpart.h"
  69 #include "vm/jit/codegen-common.h"
  70 #include "vm/jit/dseg.h"
  71 #include "vm/jit/emit-common.h"
  72 #include "vm/jit/jit.h"
  73 #include "vm/jit/methodheader.h"
  74 #include "vm/jit/parse.h"
  75 #include "vm/jit/patcher.h"
  76 #include "vm/jit/reg.h"
  77 #include "vm/jit/replace.h"
  78
  79 #if defined(ENABLE_LSRA)
  80 # include "vm/jit/allocator/lsra.h"
  81 #endif
  82
  83
  84 /* codegen *********************************************************************
  85
  86    Generates machine code.
  87
  88 *******************************************************************************/
  89
  90 bool codegen(jitdata *jd)
  91 {
  92         methodinfo         *m;
  93         codeinfo           *code;
  94         codegendata        *cd;
  95         registerdata       *rd;
  96         s4                  len, s1, s2, s3, d, disp;
  97         u2                  currentline;
  98         ptrint              a;
  99         s4                  fieldtype;
 100         varinfo            *var, *var1, *var2, *dst;
 101         basicblock         *bptr;
 102         instruction        *iptr;
 103         exceptiontable     *ex;
 104         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 105         unresolved_method  *um;
 106         builtintable_entry *bte;
 107         methoddesc         *md;
 108         rplpoint           *replacementpoint;
 109         s4                 varindex;
 110
 111         /* get required compiler data */
 112
 113         m    = jd->m;
 114         code = jd->code;
 115         cd   = jd->cd;
 116         rd   = jd->rd;
 117
 118         /* prevent compiler warnings */
 119
 120         d = 0;
 121         lm = NULL;
 122         bte = NULL;
 123
 124         {
 125         s4 i, p, t, l;
 126         s4 savedregs_num;
 127
 128         savedregs_num = 0;
 129
 130         /* space to save used callee saved registers */
 131
 132         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 133         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 134
 135         cd->stackframesize = rd->memuse + savedregs_num;
 136
 137 #if defined(ENABLE_THREADS)
 138         /* space to save argument of monitor_enter */
 139
 140         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 141                 cd->stackframesize++;
 142 #endif
 143
 144         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 145            native code e.g. libc or jni (alignment problems with
 146            movaps). */
 147
 148         if (!jd->isleafmethod || opt_verbosecall)
 149                 cd->stackframesize |= 0x1;
 150
 151         /* create method header */
 152
 153         (void) dseg_addaddress(cd, code);                      /* CodeinfoPointer */
 154         (void) dseg_adds4(cd, cd->stackframesize * 8);         /* FrameSize       */
 155
 156 #if defined(ENABLE_THREADS)
 157         /* IsSync contains the offset relative to the stack pointer for the
 158            argument of monitor_exit used in the exception handler. Since the
 159            offset could be zero and give a wrong meaning of the flag it is
 160            offset by one.
 161         */
 162
 163         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 164                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);       /* IsSync          */
 165         else
 166 #endif
 167                 (void) dseg_adds4(cd, 0);                          /* IsSync          */
 168
 169         (void) dseg_adds4(cd, jd->isleafmethod);               /* IsLeaf          */
 170         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave         */
 171         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave         */
 172
 173         (void) dseg_addlinenumbertablesize(cd);
 174
 175         (void) dseg_adds4(cd, cd->exceptiontablelength);       /* ExTableSize     */
 176
 177         /* create exception table */
 178
 179         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 180                 dseg_addtarget(cd, ex->start);
 181                 dseg_addtarget(cd, ex->end);
 182                 dseg_addtarget(cd, ex->handler);
 183                 (void) dseg_addaddress(cd, ex->catchtype.any);
 184         }
 185
 186         /* generate method profiling code */
 187
 188         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 189                 /* count frequency */
 190
 191                 M_MOV_IMM(code, REG_ITMP3);
 192                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 193
 194                 PROFILE_CYCLE_START;
 195         }
 196
 197         /* create stack frame (if necessary) */
 198
 199         if (cd->stackframesize)
 200                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 201
 202         /* save used callee saved registers */
 203
 204         p = cd->stackframesize;
 205         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 206                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 207         }
 208         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 209                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 210         }
 211
 212         /* take arguments out of register or stack frame */
 213
 214         md = m->parseddesc;
 215
 216         for (p = 0, l = 0; p < md->paramcount; p++) {
 217                 t = md->paramtypes[p].type;
 218
 219                 varindex = jd->local_map[l * 5 + t];
 220
 221                 l++;
 222                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 223                         l++;
 224
 225                 if (varindex == UNUSED)
 226                         continue;
 227
 228                 var = VAR(varindex);
 229
 230                 s1 = md->params[p].regoff;
 231
 232                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 233                         s2 = rd->argintregs[s1];
 234                         if (!md->params[p].inmemory) {           /* register arguments    */
 235                                 if (!IS_INMEMORY(var->flags)) {      /* reg arg -> register   */
 236                                         M_INTMOVE(s2, var->vv.regoff);
 237
 238                                 } else {                             /* reg arg -> spilled    */
 239                                     M_LST(s2, REG_SP, var->vv.regoff * 8);
 240                                 }
 241
 242                         } else {                                 /* stack arguments       */
 243                                 if (!IS_INMEMORY(var->flags)) {      /* stack arg -> register */
 244                                         /* + 8 for return address */
 245                                         M_LLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 246
 247                                 } else {                             /* stack arg -> spilled  */
 248                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 249                                 }
 250                         }
 251
 252                 } else {                                     /* floating args         */
 253                         if (!md->params[p].inmemory) {           /* register arguments    */
 254                                 s2 = rd->argfltregs[s1];
 255                                 if (!IS_INMEMORY(var->flags)) {      /* reg arg -> register   */
 256                                         M_FLTMOVE(s2, var->vv.regoff);
 257
 258                                 } else {                                         /* reg arg -> spilled    */
 259                                         M_DST(s2, REG_SP, var->vv.regoff * 8);
 260                                 }
 261
 262                         } else {                                 /* stack arguments       */
 263                                 if (!IS_INMEMORY(var->flags)) {      /* stack-arg -> register */
 264                                         M_DLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 265
 266                                 } else {
 267                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 268                                 }
 269                         }
 270                 }
 271         }  /* end for */
 272
 273         /* save monitorenter argument */
 274
 275 #if defined(ENABLE_THREADS)
 276         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 277                 /* stack offset for monitor argument */
 278
 279                 s1 = rd->memuse;
 280
 281                 if (opt_verbosecall) {
 282                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 283
 284                         for (p = 0; p < INT_ARG_CNT; p++)
 285                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 286
 287                         for (p = 0; p < FLT_ARG_CNT; p++)
 288                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 289
 290                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 291                 }
 292
 293                 /* decide which monitor enter function to call */
 294
 295                 if (m->flags & ACC_STATIC) {
 296                         M_MOV_IMM(&m->class->object.header, REG_A0);
 297                 }
 298                 else {
 299                         M_TEST(REG_A0);
 300                         M_BEQ(0);
 301                         codegen_add_nullpointerexception_ref(cd);
 302                 }
 303
 304                 M_AST(REG_A0, REG_SP, s1 * 8);
 305                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 306                 M_CALL(REG_ITMP1);
 307
 308                 if (opt_verbosecall) {
 309                         for (p = 0; p < INT_ARG_CNT; p++)
 310                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 311
 312                         for (p = 0; p < FLT_ARG_CNT; p++)
 313                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 314
 315                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 316                 }
 317         }
 318 #endif
 319
 320 #if !defined(NDEBUG)
 321         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 322                 emit_verbosecall_enter(jd);
 323 #endif /* !defined(NDEBUG) */
 324
 325         }
 326
 327         /* end of header generation */
 328
 329         replacementpoint = jd->code->rplpoints;
 330
 331         /* walk through all basic blocks */
 332
 333         for (bptr = jd->new_basicblocks; bptr != NULL; bptr = bptr->next) {
 334
 335                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 336
 337                 if (bptr->flags >= BBREACHED) {
 338
 339                         /* branch resolving */
 340
 341                         branchref *bref;
 342                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 343                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 344                                                                   bref->branchpos,
 345                                                                   bptr->mpc);
 346                         }
 347
 348                 /* handle replacement points */
 349
 350 #if 0
 351                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 352                         replacementpoint->pc = (u1*)(ptrint)bptr->mpc; /* will be resolved later */
 353
 354                         replacementpoint++;
 355
 356                         assert(cd->lastmcodeptr <= cd->mcodeptr);
 357                         cd->lastmcodeptr = cd->mcodeptr + 5; /* 5 byte jmp patch */
 358                 }
 359 #endif
 360
 361                 /* copy interface registers to their destination */
 362
 363                 len = bptr->indepth;
 364                 MCODECHECK(512);
 365
 366                 /* generate basicblock profiling code */
 367
 368                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 369                         /* count frequency */
 370
 371                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 372                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 373
 374                         /* if this is an exception handler, start profiling again */
 375
 376                         if (bptr->type == BBTYPE_EXH)
 377                                 PROFILE_CYCLE_START;
 378                 }
 379
 380 #if defined(ENABLE_LSRA)
 381                 if (opt_lsra) {
 382                         while (len) {
 383                                 len--;
 384                                 src = bptr->invars[len];
 385                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 386                                         if (bptr->type == BBTYPE_SBR) {
 387 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 388                                                 if (!IS_INMEMORY(src->flags))
 389                                                         d = src->vv.regoff;
 390                                                 else
 391                                                         d = REG_ITMP1;
 392                                                 M_POP(d);
 393                                                 emit_store(jd, NULL, src, d);
 394
 395                                         } else if (bptr->type == BBTYPE_EXH) {
 396 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 397                                                 if (!IS_INMEMORY(src->flags))
 398                                                         d= src->vv.regoff;
 399                                                 else
 400                                                         d=REG_ITMP1;
 401                                                 M_INTMOVE(REG_ITMP1, d);
 402                                                 emit_store(jd, NULL, src, d);
 403                                         }
 404                                 }
 405                         }
 406
 407                 } else {
 408 #endif
 409
 410                 while (len) {
 411                         len--;
 412                         varindex = bptr->invars[len];
 413                         var = VAR(varindex);
 414                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 415                                 if (bptr->type == BBTYPE_SBR) {
 416                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 417                                         M_POP(d);
 418                                         emit_store(jd, NULL, var, d);
 419
 420                                 }
 421                                 else if (bptr->type == BBTYPE_EXH) {
 422                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 423                                         M_INTMOVE(REG_ITMP1, d);
 424                                         emit_store(jd, NULL, var, d);
 425                                 }
 426
 427                         } else {
 428                                 assert((var->flags & OUTVAR));
 429                                 d = codegen_reg_of_var(0, var, REG_ITMP1);
 430                         }
 431                 }
 432 #if defined(ENABLE_LSRA)
 433                 }
 434 #endif
 435                 /* walk through all instructions */
 436
 437                 len = bptr->icount;
 438                 currentline = 0;
 439
 440                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 441                         if (iptr->line != currentline) {
 442                                 dseg_addlinenumber(cd, iptr->line);
 443                                 currentline = iptr->line;
 444                         }
 445
 446                         MCODECHECK(1024);                         /* 1KB should be enough */
 447
 448                         switch (iptr->opc) {
 449                         case ICMD_INLINE_START: /* internal ICMDs                         */
 450                         case ICMD_INLINE_END:
 451                                 break;
 452
 453                         case ICMD_NOP:    /* ...  ==> ...                                 */
 454                                 break;
 455
 456                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 457
 458                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 459                                 M_TEST(s1);
 460                                 M_BEQ(0);
 461                                 codegen_add_nullpointerexception_ref(cd);
 462                                 break;
 463
 464                 /* constant operations ************************************************/
 465
 466                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 467
 468                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 469                         ICONST(d, iptr->sx.val.i);
 470                         emit_store_dst(jd, iptr, d);
 471                         break;
 472
 473                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 474
 475                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 476                         LCONST(d, iptr->sx.val.l);
 477                         emit_store_dst(jd, iptr, d);
 478                         break;
 479
 480                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 481
 482                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 483                         disp = dseg_addfloat(cd, iptr->sx.val.f);
 484                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 485                         emit_store_dst(jd, iptr, d);
 486                         break;
 487
 488                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 489
 490                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 491                         disp = dseg_adddouble(cd, iptr->sx.val.d);
 492                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 493                         emit_store_dst(jd, iptr, d);
 494                         break;
 495
 496                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 497
 498                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 499
 500                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 501 /*                              PROFILE_CYCLE_STOP; */
 502
 503                                 codegen_addpatchref(cd, PATCHER_aconst,
 504                                                                         iptr->sx.val.c.ref, 0);
 505
 506                                 if (opt_showdisassemble) {
 507                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
 508                                 }
 509
 510 /*                              PROFILE_CYCLE_START; */
 511
 512                                 M_MOV_IMM(NULL, d);
 513
 514                         } else {
 515                                 if (iptr->sx.val.anyptr == 0)
 516                                         M_CLR(d);
 517                                 else
 518                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 519                         }
 520                         emit_store_dst(jd, iptr, d);
 521                         break;
 522
 523
 524                 /* load/store operations **********************************************/
 525
 526                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 527                 case ICMD_ALOAD:      /* s1 = local variable                          */
 528                 case ICMD_LLOAD:
 529                 case ICMD_FLOAD:
 530                 case ICMD_DLOAD:
 531                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 532                 case ICMD_ASTORE:     /* dst = local variable                         */
 533                 case ICMD_LSTORE:
 534                 case ICMD_FSTORE:
 535                 case ICMD_DSTORE:
 536
 537                         emit_copy(jd, iptr, VAROP(iptr->s1), VAROP(iptr->dst));
 538                         break;
 539
 540                 /* pop/copy/move operations *******************************************/
 541
 542                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 543
 544                 case ICMD_POP:        /* ..., value  ==> ...                          */
 545                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 546
 547                         break;
 548
 549                 case ICMD_COPY:
 550                 case ICMD_MOVE:
 551
 552                         M_COPY(iptr->s1.varindex, iptr->dst.varindex);
 553                         break;
 554
 555                 /* integer operations *************************************************/
 556
 557                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 558
 559                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 560                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 561                         M_INTMOVE(s1, d);
 562                         M_INEG(d);
 563                         emit_store_dst(jd, iptr, d);
 564                         break;
 565
 566                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 567
 568                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 569                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 570                         M_INTMOVE(s1, d);
 571                         M_LNEG(d);
 572                         emit_store_dst(jd, iptr, d);
 573                         break;
 574
 575                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 576
 577                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 578                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 579                         M_ISEXT(s1, d);
 580                         emit_store_dst(jd, iptr, d);
 581                         break;
 582
 583                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 584
 585                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 586                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 587                         M_IMOV(s1, d);
 588                         emit_store_dst(jd, iptr, d);
 589                         break;
 590
 591                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 592
 593                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 594                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 595                         M_BSEXT(s1, d);
 596                         emit_store_dst(jd, iptr, d);
 597                         break;
 598
 599                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 600
 601                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 602                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 603                         M_CZEXT(s1, d);
 604                         emit_store_dst(jd, iptr, d);
 605                         break;
 606
 607                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 608
 609                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 610                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 611                         M_SSEXT(s1, d);
 612                         emit_store_dst(jd, iptr, d);
 613                         break;
 614
 615
 616                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 617
 618                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 619                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 620                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 621                         if (s2 == d)
 622                                 M_IADD(s1, d);
 623                         else {
 624                                 M_INTMOVE(s1, d);
 625                                 M_IADD(s2, d);
 626                         }
 627                         emit_store_dst(jd, iptr, d);
 628                         break;
 629
 630                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 631                                       /* sx.val.i = constant                             */
 632
 633                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 634                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 635                         M_INTMOVE(s1, d);
 636                         M_IADD_IMM(iptr->sx.val.i, d);
 637                         emit_store_dst(jd, iptr, d);
 638                         break;
 639
 640                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 641
 642                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 643                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 644                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 645                         if (s2 == d)
 646                                 M_LADD(s1, d);
 647                         else {
 648                                 M_INTMOVE(s1, d);
 649                                 M_LADD(s2, d);
 650                         }
 651                         emit_store_dst(jd, iptr, d);
 652                         break;
 653
 654                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 655                                       /* sx.val.l = constant                             */
 656
 657                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 658                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 659                         M_INTMOVE(s1, d);
 660                         if (IS_IMM32(iptr->sx.val.l))
 661                                 M_LADD_IMM(iptr->sx.val.l, d);
 662                         else {
 663                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 664                                 M_LADD(REG_ITMP2, d);
 665                         }
 666                         emit_store_dst(jd, iptr, d);
 667                         break;
 668
 669                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 670
 671                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 672                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 673                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 674                         if (s2 == d) {
 675                                 M_INTMOVE(s1, REG_ITMP1);
 676                                 M_ISUB(s2, REG_ITMP1);
 677                                 M_INTMOVE(REG_ITMP1, d);
 678                         } else {
 679                                 M_INTMOVE(s1, d);
 680                                 M_ISUB(s2, d);
 681                         }
 682                         emit_store_dst(jd, iptr, d);
 683                         break;
 684
 685                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 686                                       /* sx.val.i = constant                             */
 687
 688                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 689                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 690                         M_INTMOVE(s1, d);
 691                         M_ISUB_IMM(iptr->sx.val.i, d);
 692                         emit_store_dst(jd, iptr, d);
 693                         break;
 694
 695                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 696
 697                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 698                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 699                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 700                         if (s2 == d) {
 701                                 M_INTMOVE(s1, REG_ITMP1);
 702                                 M_LSUB(s2, REG_ITMP1);
 703                                 M_INTMOVE(REG_ITMP1, d);
 704                         } else {
 705                                 M_INTMOVE(s1, d);
 706                                 M_LSUB(s2, d);
 707                         }
 708                         emit_store_dst(jd, iptr, d);
 709                         break;
 710
 711                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 712                                       /* sx.val.l = constant                             */
 713
 714                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 715                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 716                         M_INTMOVE(s1, d);
 717                         if (IS_IMM32(iptr->sx.val.l))
 718                                 M_LSUB_IMM(iptr->sx.val.l, d);
 719                         else {
 720                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 721                                 M_LSUB(REG_ITMP2, d);
 722                         }
 723                         emit_store_dst(jd, iptr, d);
 724                         break;
 725
 726                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 727
 728                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 729                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 730                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 731                         if (s2 == d)
 732                                 M_IMUL(s1, d);
 733                         else {
 734                                 M_INTMOVE(s1, d);
 735                                 M_IMUL(s2, d);
 736                         }
 737                         emit_store_dst(jd, iptr, d);
 738                         break;
 739
 740                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 741                                       /* sx.val.i = constant                             */
 742
 743                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 744                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 745                         if (iptr->sx.val.i == 2) {
 746                                 M_INTMOVE(s1, d);
 747                                 M_ISLL_IMM(1, d);
 748                         } else
 749                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 750                         emit_store_dst(jd, iptr, d);
 751                         break;
 752
 753                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 754
 755                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 756                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 757                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 758                         if (s2 == d)
 759                                 M_LMUL(s1, d);
 760                         else {
 761                                 M_INTMOVE(s1, d);
 762                                 M_LMUL(s2, d);
 763                         }
 764                         emit_store_dst(jd, iptr, d);
 765                         break;
 766
 767                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 768                                       /* sx.val.l = constant                             */
 769
 770                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 771                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 772                         if (IS_IMM32(iptr->sx.val.l))
 773                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 774                         else {
 775                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 776                                 M_INTMOVE(s1, d);
 777                                 M_LMUL(REG_ITMP2, d);
 778                         }
 779                         emit_store_dst(jd, iptr, d);
 780                         break;
 781
 782                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 783
 784                         var1 = VAROP(iptr->s1);
 785                         var2 = VAROP(iptr->sx.s23.s2);
 786                         dst  = VAROP(iptr->dst);
 787
 788                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 789                         if (IS_INMEMORY(var1->flags))
 790                                 M_ILD(RAX, REG_SP, var1->vv.regoff * 8);
 791                         else
 792                                 M_INTMOVE(var1->vv.regoff, RAX);
 793
 794                         if (IS_INMEMORY(var2->flags))
 795                                 M_ILD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 796                         else
 797                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 798
 799                         if (checknull) {
 800                                 M_ITEST(REG_ITMP3);
 801                                 M_BEQ(0);
 802                                 codegen_add_arithmeticexception_ref(cd);
 803                         }
 804
 805                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
 806                         emit_jcc(cd, CC_NE, 4 + 6);
 807                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
 808                         emit_jcc(cd, CC_E, 3 + 1 + 3);                /* 6 bytes */
 809
 810                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 811                         emit_cltd(cd);
 812                         emit_idivl_reg(cd, REG_ITMP3);
 813
 814                         if (IS_INMEMORY(dst->flags)) {
 815                                 emit_mov_reg_membase(cd, RAX, REG_SP, dst->vv.regoff * 8);
 816                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
 817
 818                         } else {
 819                                 M_INTMOVE(RAX, dst->vv.regoff);
 820
 821                                 if (dst->vv.regoff != RDX) {
 822                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
 823                                 }
 824                         }
 825                         break;
 826
 827                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 828
 829                         var1 = VAROP(iptr->s1);
 830                         var2 = VAROP(iptr->sx.s23.s2);
 831                         dst  = VAROP(iptr->dst);
 832
 833                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 834                         if (IS_INMEMORY(var1->flags))
 835                                 M_ILD(RAX, REG_SP, var1->vv.regoff * 8);
 836                         else
 837                                 M_INTMOVE(var1->vv.regoff, RAX);
 838
 839                         if (IS_INMEMORY(var2->flags))
 840                                 M_ILD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 841                         else
 842                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 843
 844                         if (checknull) {
 845                                 M_ITEST(REG_ITMP3);
 846                                 M_BEQ(0);
 847                                 codegen_add_arithmeticexception_ref(cd);
 848                         }
 849
 850                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 851
 852                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
 853                         emit_jcc(cd, CC_NE, 2 + 4 + 6);
 854
 855
 856                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
 857                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
 858                         emit_jcc(cd, CC_E, 1 + 3);                    /* 6 bytes */
 859
 860                         emit_cltd(cd);
 861                         emit_idivl_reg(cd, REG_ITMP3);
 862
 863                         if (IS_INMEMORY(dst->flags)) {
 864                                 emit_mov_reg_membase(cd, RDX, REG_SP, dst->vv.regoff * 8);
 865                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
 866
 867                         } else {
 868                                 M_INTMOVE(RDX, dst->vv.regoff);
 869
 870                                 if (dst->vv.regoff != RDX) {
 871                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
 872                                 }
 873                         }
 874                         break;
 875
 876                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 877                                       /* sx.val.i = constant                             */
 878
 879                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 880                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 881                         M_INTMOVE(s1, REG_ITMP1);
 882                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 883                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 884                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 885                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 886                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 887                         emit_store_dst(jd, iptr, d);
 888                         break;
 889
 890                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 891                                       /* sx.val.i = constant                             */
 892
 893                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 894                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 895                         M_INTMOVE(s1, REG_ITMP1);
 896                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 897                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 898                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 899                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 900                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 901                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 902                         emit_store_dst(jd, iptr, d);
 903                         break;
 904
 905
 906                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 907
 908                         var1 = VAROP(iptr->s1);
 909                         var2 = VAROP(iptr->sx.s23.s2);
 910                         dst  = VAROP(iptr->dst);
 911
 912                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 913
 914                 if (IS_INMEMORY(var1->flags))
 915                                 M_LLD(RAX, REG_SP, var1->vv.regoff * 8);
 916                         else
 917                                 M_INTMOVE(var1->vv.regoff, RAX);
 918
 919                         if (IS_INMEMORY(var2->flags))
 920                                 M_LLD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 921                         else
 922                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 923
 924                         if (checknull) {
 925                                 M_TEST(REG_ITMP3);
 926                                 M_BEQ(0);
 927                                 codegen_add_arithmeticexception_ref(cd);
 928                         }
 929
 930                         /* check as described in jvm spec */
 931                         disp = dseg_adds8(cd, 0x8000000000000000LL);
 932                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 933                         M_BNE(4 + 6);
 934                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
 935                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
 936
 937                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 938                         emit_cqto(cd);
 939                         emit_idiv_reg(cd, REG_ITMP3);
 940
 941                         if (IS_INMEMORY(dst->flags)) {
 942                                 M_LST(RAX, REG_SP, dst->vv.regoff * 8);
 943                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
 944
 945                         } else {
 946                                 M_INTMOVE(RAX, dst->vv.regoff);
 947
 948                                 if (dst->vv.regoff != RDX) {
 949                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
 950                                 }
 951                         }
 952                         break;
 953
 954                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 955
 956                         var1 = VAROP(iptr->s1);
 957                         var2 = VAROP(iptr->sx.s23.s2);
 958                         dst  = VAROP(iptr->dst);
 959
 960                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 961
 962                         if (IS_INMEMORY(var1->flags))
 963                                 M_LLD(REG_ITMP1, REG_SP, var1->vv.regoff * 8);
 964                         else
 965                                 M_INTMOVE(var1->vv.regoff, REG_ITMP1);
 966
 967                         if (IS_INMEMORY(var2->flags))
 968                                 M_LLD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 969                         else
 970                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 971                         /*
 972                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 973                         M_INTMOVE(s1, REG_ITMP1);
 974                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 975                         M_INTMOVE(s2, REG_ITMP3);
 976                         */
 977                         if (checknull) {
 978                                 M_ITEST(REG_ITMP3);
 979                                 M_BEQ(0);
 980                                 codegen_add_arithmeticexception_ref(cd);
 981                         }
 982
 983                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 984
 985                         /* check as described in jvm spec */
 986                         disp = dseg_adds8(cd, 0x8000000000000000LL);
 987                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 988                         M_BNE(3 + 4 + 6);
 989
 990 #if 0
 991                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
 992 #endif
 993                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 994                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
 995                         M_BEQ(2 + 3);                                          /* 6 bytes */
 996
 997                         emit_cqto(cd);
 998                         emit_idiv_reg(cd, REG_ITMP3);
 999
1000                         if (IS_INMEMORY(dst->flags)) {
1001                                 M_LST(RDX, REG_SP, dst->vv.regoff * 8);
1002                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1003
1004                         } else {
1005                                 M_INTMOVE(RDX, dst->vv.regoff);
1006
1007                                 if (dst->vv.regoff != RDX) {
1008                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1009                                 }
1010                         }
1011                         break;
1012
1013                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1014                                       /* sx.val.i = constant                             */
1015
1016                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1017                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1018                         M_INTMOVE(s1, REG_ITMP1);
1019                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1020                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
1021                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
1022                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
1023                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1024                         emit_store_dst(jd, iptr, d);
1025                         break;
1026
1027                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1028                                       /* sx.val.l = constant                             */
1029
1030                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1031                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1032                         M_INTMOVE(s1, REG_ITMP1);
1033                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1034                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
1035                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
1036                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
1037                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
1038                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1039                         emit_store_dst(jd, iptr, d);
1040                         break;
1041
1042                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1043
1044                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1045                         emit_ishift(jd, SHIFT_SHL, iptr);
1046                         break;
1047
1048                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1049                                       /* sx.val.i = constant                             */
1050
1051                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1052                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1053                         M_INTMOVE(s1, d);
1054                         M_ISLL_IMM(iptr->sx.val.i, d);
1055                         emit_store_dst(jd, iptr, d);
1056                         break;
1057
1058                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1059
1060                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1061                         emit_ishift(jd, SHIFT_SAR, iptr);
1062                         break;
1063
1064                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1065                                       /* sx.val.i = constant                             */
1066
1067                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1068                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1069                         M_INTMOVE(s1, d);
1070                         M_ISRA_IMM(iptr->sx.val.i, d);
1071                         emit_store_dst(jd, iptr, d);
1072                         break;
1073
1074                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1075
1076                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1077                         emit_ishift(jd, SHIFT_SHR, iptr);
1078                         break;
1079
1080                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1081                                       /* sx.val.i = constant                             */
1082
1083                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1084                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1085                         M_INTMOVE(s1, d);
1086                         M_ISRL_IMM(iptr->sx.val.i, d);
1087                         emit_store_dst(jd, iptr, d);
1088                         break;
1089
1090                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1091
1092                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1093                         emit_lshift(jd, SHIFT_SHL, iptr);
1094                         break;
1095
1096         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1097                                           /* sx.val.i = constant                             */
1098
1099                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1100                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1101                         M_INTMOVE(s1, d);
1102                         M_LSLL_IMM(iptr->sx.val.i, d);
1103                         emit_store_dst(jd, iptr, d);
1104                         break;
1105
1106                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1107
1108                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1109                         emit_lshift(jd, SHIFT_SAR, iptr);
1110                         break;
1111
1112                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1113                                       /* sx.val.i = constant                             */
1114
1115                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1116                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1117                         M_INTMOVE(s1, d);
1118                         M_LSRA_IMM(iptr->sx.val.i, d);
1119                         emit_store_dst(jd, iptr, d);
1120                         break;
1121
1122                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1123
1124                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1125                         emit_lshift(jd, SHIFT_SHR, iptr);
1126                         break;
1127
1128                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1129                                       /* sx.val.l = constant                             */
1130
1131                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1132                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1133                         M_INTMOVE(s1, d);
1134                         M_LSRL_IMM(iptr->sx.val.i, d);
1135                         emit_store_dst(jd, iptr, d);
1136                         break;
1137
1138                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1139
1140                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1141                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1142                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1143                         if (s2 == d)
1144                                 M_IAND(s1, d);
1145                         else {
1146                                 M_INTMOVE(s1, d);
1147                                 M_IAND(s2, d);
1148                         }
1149                         emit_store_dst(jd, iptr, d);
1150                         break;
1151
1152                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1153                                       /* sx.val.i = constant                             */
1154
1155                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1156                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1157                         M_INTMOVE(s1, d);
1158                         M_IAND_IMM(iptr->sx.val.i, d);
1159                         emit_store_dst(jd, iptr, d);
1160                         break;
1161
1162                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1163
1164                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1165                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1166                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1167                         if (s2 == d)
1168                                 M_LAND(s1, d);
1169                         else {
1170                                 M_INTMOVE(s1, d);
1171                                 M_LAND(s2, d);
1172                         }
1173                         emit_store_dst(jd, iptr, d);
1174                         break;
1175
1176                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1177                                       /* sx.val.l = constant                             */
1178
1179                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1180                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1181                         M_INTMOVE(s1, d);
1182                         if (IS_IMM32(iptr->sx.val.l))
1183                                 M_LAND_IMM(iptr->sx.val.l, d);
1184                         else {
1185                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1186                                 M_LAND(REG_ITMP2, d);
1187                         }
1188                         emit_store_dst(jd, iptr, d);
1189                         break;
1190
1191                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1192
1193                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1194                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1195                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1196                         if (s2 == d)
1197                                 M_IOR(s1, d);
1198                         else {
1199                                 M_INTMOVE(s1, d);
1200                                 M_IOR(s2, d);
1201                         }
1202                         emit_store_dst(jd, iptr, d);
1203                         break;
1204
1205                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1206                                       /* sx.val.i = constant                             */
1207
1208                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1209                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1210                         M_INTMOVE(s1, d);
1211                         M_IOR_IMM(iptr->sx.val.i, d);
1212                         emit_store_dst(jd, iptr, d);
1213                         break;
1214
1215                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1216
1217                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1218                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1219                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1220                         if (s2 == d)
1221                                 M_LOR(s1, d);
1222                         else {
1223                                 M_INTMOVE(s1, d);
1224                                 M_LOR(s2, d);
1225                         }
1226                         emit_store_dst(jd, iptr, d);
1227                         break;
1228
1229                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1230                                       /* sx.val.l = constant                             */
1231
1232                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1233                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1234                         M_INTMOVE(s1, d);
1235                         if (IS_IMM32(iptr->sx.val.l))
1236                                 M_LOR_IMM(iptr->sx.val.l, d);
1237                         else {
1238                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1239                                 M_LOR(REG_ITMP2, d);
1240                         }
1241                         emit_store_dst(jd, iptr, d);
1242                         break;
1243
1244                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1245
1246                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1247                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1248                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1249                         if (s2 == d)
1250                                 M_IXOR(s1, d);
1251                         else {
1252                                 M_INTMOVE(s1, d);
1253                                 M_IXOR(s2, d);
1254                         }
1255                         emit_store_dst(jd, iptr, d);
1256                         break;
1257
1258                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1259                                       /* sx.val.i = constant                             */
1260
1261                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1262                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1263                         M_INTMOVE(s1, d);
1264                         M_IXOR_IMM(iptr->sx.val.i, d);
1265                         emit_store_dst(jd, iptr, d);
1266                         break;
1267
1268                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1269
1270                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1271                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1272                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1273                         if (s2 == d)
1274                                 M_LXOR(s1, d);
1275                         else {
1276                                 M_INTMOVE(s1, d);
1277                                 M_LXOR(s2, d);
1278                         }
1279                         emit_store_dst(jd, iptr, d);
1280                         break;
1281
1282                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1283                                       /* sx.val.l = constant                             */
1284
1285                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1286                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1287                         M_INTMOVE(s1, d);
1288                         if (IS_IMM32(iptr->sx.val.l))
1289                                 M_LXOR_IMM(iptr->sx.val.l, d);
1290                         else {
1291                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1292                                 M_LXOR(REG_ITMP2, d);
1293                         }
1294                         emit_store_dst(jd, iptr, d);
1295                         break;
1296
1297
1298                 case ICMD_IINC:      /* ..., value  ==> ..., value + constant         */
1299                                      /* s1.localindex = variable, sx.val.i = constant */
1300
1301                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1302                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1303
1304                         /* Using inc and dec is not faster than add (tested with
1305                            sieve). */
1306
1307                         M_INTMOVE(s1, d);
1308                         M_IADD_IMM(iptr->sx.val.i, d);
1309
1310                         emit_store_dst(jd, iptr, d);
1311                         break;
1312
1313
1314                 /* floating operations ************************************************/
1315
1316                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1317
1318                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1319                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1320                         disp = dseg_adds4(cd, 0x80000000);
1321                         M_FLTMOVE(s1, d);
1322                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1323                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1324                         emit_store_dst(jd, iptr, d);
1325                         break;
1326
1327                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1328
1329                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1330                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1331                         disp = dseg_adds8(cd, 0x8000000000000000);
1332                         M_FLTMOVE(s1, d);
1333                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1334                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1335                         emit_store_dst(jd, iptr, d);
1336                         break;
1337
1338                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1339
1340                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1341                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1342                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1343                         if (s2 == d)
1344                                 M_FADD(s1, d);
1345                         else {
1346                                 M_FLTMOVE(s1, d);
1347                                 M_FADD(s2, d);
1348                         }
1349                         emit_store_dst(jd, iptr, d);
1350                         break;
1351
1352                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1353
1354                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1355                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1356                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1357                         if (s2 == d)
1358                                 M_DADD(s1, d);
1359                         else {
1360                                 M_FLTMOVE(s1, d);
1361                                 M_DADD(s2, d);
1362                         }
1363                         emit_store_dst(jd, iptr, d);
1364                         break;
1365
1366                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1367
1368                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1369                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1370                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1371                         if (s2 == d) {
1372                                 M_FLTMOVE(s2, REG_FTMP2);
1373                                 s2 = REG_FTMP2;
1374                         }
1375                         M_FLTMOVE(s1, d);
1376                         M_FSUB(s2, d);
1377                         emit_store_dst(jd, iptr, d);
1378                         break;
1379
1380                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1381
1382                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1383                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1384                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1385                         if (s2 == d) {
1386                                 M_FLTMOVE(s2, REG_FTMP2);
1387                                 s2 = REG_FTMP2;
1388                         }
1389                         M_FLTMOVE(s1, d);
1390                         M_DSUB(s2, d);
1391                         emit_store_dst(jd, iptr, d);
1392                         break;
1393
1394                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1395
1396                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1397                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1398                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1399                         if (s2 == d)
1400                                 M_FMUL(s1, d);
1401                         else {
1402                                 M_FLTMOVE(s1, d);
1403                                 M_FMUL(s2, d);
1404                         }
1405                         emit_store_dst(jd, iptr, d);
1406                         break;
1407
1408                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1409
1410                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1411                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1412                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1413                         if (s2 == d)
1414                                 M_DMUL(s1, d);
1415                         else {
1416                                 M_FLTMOVE(s1, d);
1417                                 M_DMUL(s2, d);
1418                         }
1419                         emit_store_dst(jd, iptr, d);
1420                         break;
1421
1422                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1423
1424                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1425                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1426                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1427                         if (s2 == d) {
1428                                 M_FLTMOVE(s2, REG_FTMP2);
1429                                 s2 = REG_FTMP2;
1430                         }
1431                         M_FLTMOVE(s1, d);
1432                         M_FDIV(s2, d);
1433                         emit_store_dst(jd, iptr, d);
1434                         break;
1435
1436                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1437
1438                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1439                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1440                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1441                         if (s2 == d) {
1442                                 M_FLTMOVE(s2, REG_FTMP2);
1443                                 s2 = REG_FTMP2;
1444                         }
1445                         M_FLTMOVE(s1, d);
1446                         M_DDIV(s2, d);
1447                         emit_store_dst(jd, iptr, d);
1448                         break;
1449
1450                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1451
1452                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1453                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1454                         M_CVTIF(s1, d);
1455                         emit_store_dst(jd, iptr, d);
1456                         break;
1457
1458                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1459
1460                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1461                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1462                         M_CVTID(s1, d);
1463                         emit_store_dst(jd, iptr, d);
1464                         break;
1465
1466                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1467
1468                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1469                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1470                         M_CVTLF(s1, d);
1471                         emit_store_dst(jd, iptr, d);
1472                         break;
1473
1474                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1475
1476                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1477                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1478                         M_CVTLD(s1, d);
1479                         emit_store_dst(jd, iptr, d);
1480                         break;
1481
1482                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1483
1484                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1485                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1486                         M_CVTFI(s1, d);
1487                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1488                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1489                                 ((REG_RESULT == d) ? 0 : 3);
1490                         M_BNE(disp);
1491                         M_FLTMOVE(s1, REG_FTMP1);
1492                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1493                         M_CALL(REG_ITMP2);
1494                         M_INTMOVE(REG_RESULT, d);
1495                         emit_store_dst(jd, iptr, d);
1496                         break;
1497
1498                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1499
1500                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1501                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1502                         M_CVTDI(s1, d);
1503                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1504                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1505                                 ((REG_RESULT == d) ? 0 : 3);
1506                         M_BNE(disp);
1507                         M_FLTMOVE(s1, REG_FTMP1);
1508                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1509                         M_CALL(REG_ITMP2);
1510                         M_INTMOVE(REG_RESULT, d);
1511                         emit_store_dst(jd, iptr, d);
1512                         break;
1513
1514                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1515
1516                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1517                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1518                         M_CVTFL(s1, d);
1519                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1520                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1521                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1522                                 ((REG_RESULT == d) ? 0 : 3);
1523                         M_BNE(disp);
1524                         M_FLTMOVE(s1, REG_FTMP1);
1525                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1526                         M_CALL(REG_ITMP2);
1527                         M_INTMOVE(REG_RESULT, d);
1528                         emit_store_dst(jd, iptr, d);
1529                         break;
1530
1531                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1532
1533                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1534                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1535                         M_CVTDL(s1, d);
1536                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1537                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1538                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1539                                 ((REG_RESULT == d) ? 0 : 3);
1540                         M_BNE(disp);
1541                         M_FLTMOVE(s1, REG_FTMP1);
1542                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1543                         M_CALL(REG_ITMP2);
1544                         M_INTMOVE(REG_RESULT, d);
1545                         emit_store_dst(jd, iptr, d);
1546                         break;
1547
1548                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1549
1550                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1551                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1552                         M_CVTFD(s1, d);
1553                         emit_store_dst(jd, iptr, d);
1554                         break;
1555
1556                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1557
1558                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1559                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1560                         M_CVTDF(s1, d);
1561                         emit_store_dst(jd, iptr, d);
1562                         break;
1563
1564                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1565                                           /* == => 0, < => 1, > => -1 */
1566
1567                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1568                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1569                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1570                         M_CLR(d);
1571                         M_MOV_IMM(1, REG_ITMP1);
1572                         M_MOV_IMM(-1, REG_ITMP2);
1573                         emit_ucomiss_reg_reg(cd, s1, s2);
1574                         M_CMOVB(REG_ITMP1, d);
1575                         M_CMOVA(REG_ITMP2, d);
1576                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1577                         emit_store_dst(jd, iptr, d);
1578                         break;
1579
1580                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1581                                           /* == => 0, < => 1, > => -1 */
1582
1583                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1584                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1585                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1586                         M_CLR(d);
1587                         M_MOV_IMM(1, REG_ITMP1);
1588                         M_MOV_IMM(-1, REG_ITMP2);
1589                         emit_ucomiss_reg_reg(cd, s1, s2);
1590                         M_CMOVB(REG_ITMP1, d);
1591                         M_CMOVA(REG_ITMP2, d);
1592                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1593                         emit_store_dst(jd, iptr, d);
1594                         break;
1595
1596                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1597                                           /* == => 0, < => 1, > => -1 */
1598
1599                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1600                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1601                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1602                         M_CLR(d);
1603                         M_MOV_IMM(1, REG_ITMP1);
1604                         M_MOV_IMM(-1, REG_ITMP2);
1605                         emit_ucomisd_reg_reg(cd, s1, s2);
1606                         M_CMOVB(REG_ITMP1, d);
1607                         M_CMOVA(REG_ITMP2, d);
1608                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1609                         emit_store_dst(jd, iptr, d);
1610                         break;
1611
1612                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1613                                           /* == => 0, < => 1, > => -1 */
1614
1615                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1616                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1617                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1618                         M_CLR(d);
1619                         M_MOV_IMM(1, REG_ITMP1);
1620                         M_MOV_IMM(-1, REG_ITMP2);
1621                         emit_ucomisd_reg_reg(cd, s1, s2);
1622                         M_CMOVB(REG_ITMP1, d);
1623                         M_CMOVA(REG_ITMP2, d);
1624                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1625                         emit_store_dst(jd, iptr, d);
1626                         break;
1627
1628
1629                 /* memory operations **************************************************/
1630
1631                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1632
1633                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1634                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1635                         gen_nullptr_check(s1);
1636                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1637                         emit_store_dst(jd, iptr, d);
1638                         break;
1639
1640                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1641
1642                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1643                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1644                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1645                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1646                                 gen_nullptr_check(s1);
1647                                 gen_bound_check;
1648                         }
1649                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1650                         emit_store_dst(jd, iptr, d);
1651                         break;
1652
1653                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1654
1655                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1656                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1657                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1658                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1659                                 gen_nullptr_check(s1);
1660                                 gen_bound_check;
1661                         }
1662                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1663                         emit_store_dst(jd, iptr, d);
1664                         break;
1665
1666                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1667
1668                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1669                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1670                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1671                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1672                                 gen_nullptr_check(s1);
1673                                 gen_bound_check;
1674                         }
1675                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1676                         emit_store_dst(jd, iptr, d);
1677                         break;
1678
1679                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1680
1681                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1682                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1683                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1684                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1685                                 gen_nullptr_check(s1);
1686                                 gen_bound_check;
1687                         }
1688                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1689                         emit_store_dst(jd, iptr, d);
1690                         break;
1691
1692                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1693
1694                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1695                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1696                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1697                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1698                                 gen_nullptr_check(s1);
1699                                 gen_bound_check;
1700                         }
1701                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1702                         emit_store_dst(jd, iptr, d);
1703                         break;
1704
1705                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1706
1707                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1708                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1709                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1710                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1711                                 gen_nullptr_check(s1);
1712                                 gen_bound_check;
1713                         }
1714                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1715                         emit_store_dst(jd, iptr, d);
1716                         break;
1717
1718                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1719
1720                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1721                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1722                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1723                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1724                                 gen_nullptr_check(s1);
1725                                 gen_bound_check;
1726                         }
1727                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1728                         emit_store_dst(jd, iptr, d);
1729                         break;
1730
1731                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1732
1733                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1734                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1735                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1736                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1737                                 gen_nullptr_check(s1);
1738                                 gen_bound_check;
1739                         }
1740                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1741                         emit_store_dst(jd, iptr, d);
1742                         break;
1743
1744
1745                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1746
1747                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1748                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1749                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1750                                 gen_nullptr_check(s1);
1751                                 gen_bound_check;
1752                         }
1753                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1754                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1755                         break;
1756
1757                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1758
1759                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1760                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1761                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1762                                 gen_nullptr_check(s1);
1763                                 gen_bound_check;
1764                         }
1765                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1766                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1767                         break;
1768
1769                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1770
1771                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1772                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1773                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1774                                 gen_nullptr_check(s1);
1775                                 gen_bound_check;
1776                         }
1777                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1778                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1779                         break;
1780
1781                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1782
1783                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1784                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1785                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1786                                 gen_nullptr_check(s1);
1787                                 gen_bound_check;
1788                         }
1789                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1790                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
1791                         break;
1792
1793                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1794
1795                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1796                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1797                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1798                                 gen_nullptr_check(s1);
1799                                 gen_bound_check;
1800                         }
1801                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1802                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
1803                         break;
1804
1805                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1806
1807                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1808                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1809                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1810                                 gen_nullptr_check(s1);
1811                                 gen_bound_check;
1812                         }
1813                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1814                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
1815                         break;
1816
1817                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1818
1819                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1820                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1821                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1822                                 gen_nullptr_check(s1);
1823                                 gen_bound_check;
1824                         }
1825                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1826                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
1827                         break;
1828
1829                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1830
1831                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1832                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1833                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1834                                 gen_nullptr_check(s1);
1835                                 gen_bound_check;
1836                         }
1837                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1838
1839                         M_MOV(s1, REG_A0);
1840                         M_MOV(s3, REG_A1);
1841                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
1842                         M_CALL(REG_ITMP1);
1843                         M_TEST(REG_RESULT);
1844                         M_BEQ(0);
1845                         codegen_add_arraystoreexception_ref(cd);
1846
1847                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1848                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1849                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1850                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1851                         break;
1852
1853
1854                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1855
1856                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1857                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1858                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1859                                 gen_nullptr_check(s1);
1860                                 gen_bound_check;
1861                         }
1862                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1863                         break;
1864
1865                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1866
1867                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1868                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1869                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1870                                 gen_nullptr_check(s1);
1871                                 gen_bound_check;
1872                         }
1873                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray, data[0]), s1, s2, 1);
1874                         break;
1875
1876                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1877
1878                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1879                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1880                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1881                                 gen_nullptr_check(s1);
1882                                 gen_bound_check;
1883                         }
1884                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1885                         break;
1886
1887                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1888
1889                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1890                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1891                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1892                                 gen_nullptr_check(s1);
1893                                 gen_bound_check;
1894                         }
1895                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray, data[0]), s1, s2, 2);
1896                         break;
1897
1898                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1899
1900                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1901                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1902                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1903                                 gen_nullptr_check(s1);
1904                                 gen_bound_check;
1905                         }
1906
1907                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1908                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1909                         } else {
1910                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1911                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
1912                         }
1913                         break;
1914
1915                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1916
1917                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1918                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1919                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1920                                 gen_nullptr_check(s1);
1921                                 gen_bound_check;
1922                         }
1923                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1924                         break;
1925
1926
1927                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1928
1929                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1930                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
1931
1932                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1933                                 disp = dseg_addaddress(cd, NULL);
1934                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1935
1936                                 /* must be calculated before codegen_addpatchref */
1937
1938                                 if (opt_showdisassemble)
1939                                         disp -= PATCHER_CALL_SIZE;
1940
1941 /*                              PROFILE_CYCLE_STOP; */
1942
1943                                 codegen_addpatchref(cd, PATCHER_get_putstatic, uf, disp);
1944
1945                                 if (opt_showdisassemble) {
1946                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
1947                                 }
1948
1949 /*                              PROFILE_CYCLE_START; */
1950                         }
1951                         else {
1952                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
1953
1954                                 fieldtype = fi->type;
1955
1956                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1957                                         PROFILE_CYCLE_STOP;
1958
1959                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
1960
1961                                         if (opt_showdisassemble) {
1962                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
1963                                         }
1964
1965                                         PROFILE_CYCLE_START;
1966                                 }
1967
1968                                 disp = dseg_addaddress(cd, &(fi->value));
1969                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1970                         }
1971
1972                         /* This approach is much faster than moving the field
1973                            address inline into a register. */
1974
1975                         M_ALD(REG_ITMP1, RIP, disp);
1976
1977                         switch (fieldtype) {
1978                         case TYPE_INT:
1979                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1980                                 M_ILD(d, REG_ITMP1, 0);
1981                                 break;
1982                         case TYPE_LNG:
1983                         case TYPE_ADR:
1984                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1985                                 M_LLD(d, REG_ITMP1, 0);
1986                                 break;
1987                         case TYPE_FLT:
1988                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1989                                 M_FLD(d, REG_ITMP1, 0);
1990                                 break;
1991                         case TYPE_DBL:
1992                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1993                                 M_DLD(d, REG_ITMP1, 0);
1994                                 break;
1995                         }
1996                         emit_store_dst(jd, iptr, d);
1997                         break;
1998
1999                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2000
2001                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2002                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2003
2004                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2005                                 disp = dseg_addaddress(cd, NULL);
2006                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2007
2008                                 /* must be calculated before codegen_addpatchref */
2009
2010                                 if (opt_showdisassemble)
2011                                         disp -= PATCHER_CALL_SIZE;
2012
2013 /*                              PROFILE_CYCLE_STOP; */
2014
2015                                 codegen_addpatchref(cd, PATCHER_get_putstatic, uf, disp);
2016
2017                                 if (opt_showdisassemble) {
2018                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2019                                 }
2020
2021 /*                              PROFILE_CYCLE_START; */
2022                         }
2023                         else {
2024                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2025
2026                                 fieldtype = fi->type;
2027
2028                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2029                                         PROFILE_CYCLE_STOP;
2030
2031                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2032
2033                                         if (opt_showdisassemble) {
2034                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2035                                                 disp -= PATCHER_CALL_SIZE;
2036                                         }
2037
2038                                         PROFILE_CYCLE_START;
2039                                 }
2040
2041                                 disp = dseg_addaddress(cd, &(fi->value));
2042                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2043                         }
2044
2045                         /* This approach is much faster than moving the field
2046                            address inline into a register. */
2047
2048                         M_ALD(REG_ITMP1, RIP, disp);
2049
2050                         switch (fieldtype) {
2051                         case TYPE_INT:
2052                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2053                                 M_IST(s1, REG_ITMP1, 0);
2054                                 break;
2055                         case TYPE_LNG:
2056                         case TYPE_ADR:
2057                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2058                                 M_LST(s1, REG_ITMP1, 0);
2059                                 break;
2060                         case TYPE_FLT:
2061                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
2062                                 M_FST(s1, REG_ITMP1, 0);
2063                                 break;
2064                         case TYPE_DBL:
2065                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
2066                                 M_DST(s1, REG_ITMP1, 0);
2067                                 break;
2068                         }
2069                         break;
2070
2071                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2072                                           /* val = value (in current instruction)     */
2073                                           /* following NOP)                           */
2074
2075                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2076                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2077
2078                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2079                                 disp = dseg_addaddress(cd, NULL);
2080                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2081
2082                                 /* must be calculated before codegen_addpatchref */
2083
2084                                 if (opt_showdisassemble)
2085                                         disp -= PATCHER_CALL_SIZE;
2086
2087
2088 /*                              PROFILE_CYCLE_STOP; */
2089
2090                                 codegen_addpatchref(cd, PATCHER_get_putstatic, uf, disp);
2091
2092                                 if (opt_showdisassemble) {
2093                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2094                                 }
2095
2096 /*                              PROFILE_CYCLE_START; */
2097                         }
2098                         else {
2099                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2100
2101                                 fieldtype = fi->type;
2102
2103                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2104                                         PROFILE_CYCLE_STOP;
2105
2106                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2107
2108                                         if (opt_showdisassemble) {
2109                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2110                                         }
2111
2112                                         PROFILE_CYCLE_START;
2113                                 }
2114
2115                                 disp = dseg_addaddress(cd, &(fi->value));
2116                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2117                         }
2118
2119                         /* This approach is much faster than moving the field
2120                            address inline into a register. */
2121
2122                         M_ALD(REG_ITMP1, RIP, disp);
2123
2124                         switch (fieldtype) {
2125                         case TYPE_INT:
2126                         case TYPE_FLT:
2127                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2128                                 break;
2129                         case TYPE_LNG:
2130                         case TYPE_ADR:
2131                         case TYPE_DBL:
2132                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
2133                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2134                                 else {
2135                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2136                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
2137                                 }
2138                                 break;
2139                         }
2140                         break;
2141
2142                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2143
2144                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2145                         gen_nullptr_check(s1);
2146
2147                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2148                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2149
2150 /*                              PROFILE_CYCLE_STOP; */
2151
2152                                 codegen_addpatchref(cd, PATCHER_get_putfield, uf, 0);
2153
2154                                 if (opt_showdisassemble) {
2155                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2156                                 }
2157
2158 /*                              PROFILE_CYCLE_START; */
2159
2160                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2161                                 disp = 0;
2162                         }
2163                         else {
2164                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2165
2166                                 fieldtype = fi->type;
2167                                 disp = fi->offset;
2168                         }
2169
2170                         switch (fieldtype) {
2171                         case TYPE_INT:
2172                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
2173                                 M_ILD32(d, s1, disp);
2174                                 break;
2175                         case TYPE_LNG:
2176                         case TYPE_ADR:
2177                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
2178                                 M_LLD32(d, s1, disp);
2179                                 break;
2180                         case TYPE_FLT:
2181                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2182                                 M_FLD32(d, s1, disp);
2183                                 break;
2184                         case TYPE_DBL:
2185                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2186                                 M_DLD32(d, s1, disp);
2187                                 break;
2188                         }
2189                         emit_store_dst(jd, iptr, d);
2190                         break;
2191
2192                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2193
2194                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2195                         gen_nullptr_check(s1);
2196
2197                         s2 = emit_load_s2(jd, iptr, REG_IFTMP);
2198
2199                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2200                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2201
2202 /*                              PROFILE_CYCLE_STOP; */
2203
2204                                 codegen_addpatchref(cd, PATCHER_get_putfield, uf, 0);
2205
2206                                 if (opt_showdisassemble) {
2207                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2208                                 }
2209
2210 /*                              PROFILE_CYCLE_START; */
2211
2212                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2213                                 disp = 0;
2214                         }
2215                         else {
2216                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2217
2218                                 fieldtype = fi->type;
2219                                 disp = fi->offset;
2220                         }
2221
2222                         switch (fieldtype) {
2223                         case TYPE_INT:
2224                                 M_IST32(s2, s1, disp);
2225                                 break;
2226                         case TYPE_LNG:
2227                         case TYPE_ADR:
2228                                 M_LST32(s2, s1, disp);
2229                                 break;
2230                         case TYPE_FLT:
2231                                 M_FST32(s2, s1, disp);
2232                                 break;
2233                         case TYPE_DBL:
2234                                 M_DST32(s2, s1, disp);
2235                                 break;
2236                         }
2237                         break;
2238
2239                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2240                                           /* val = value (in current instruction)     */
2241                                           /* following NOP)                           */
2242
2243                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2244                         gen_nullptr_check(s1);
2245
2246                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2247                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2248
2249 /*                              PROFILE_CYCLE_STOP; */
2250
2251                                 codegen_addpatchref(cd, PATCHER_putfieldconst, uf, 0);
2252
2253                                 if (opt_showdisassemble) {
2254                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2255                                 }
2256
2257 /*                              PROFILE_CYCLE_START; */
2258
2259                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2260                                 disp = 0;
2261                         }
2262                         else {
2263                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2264
2265                                 fieldtype = fi->type;
2266                                 disp = fi->offset;
2267                         }
2268
2269                         switch (fieldtype) {
2270                         case TYPE_INT:
2271                         case TYPE_FLT:
2272                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2273                                 break;
2274                         case TYPE_LNG:
2275                         case TYPE_ADR:
2276                         case TYPE_DBL:
2277                                 /* XXX why no check for IS_IMM32? */
2278                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2279                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2280                                 break;
2281                         }
2282                         break;
2283
2284
2285                 /* branch operations **************************************************/
2286
2287                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2288
2289                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2290                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2291
2292                         PROFILE_CYCLE_STOP;
2293
2294 #ifdef ENABLE_VERIFIER
2295                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2296                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2297                                                                         iptr->sx.s23.s2.uc, 0);
2298
2299                                 if (opt_showdisassemble) {
2300                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2301                                 }
2302                         }
2303 #endif /* ENABLE_VERIFIER */
2304
2305                         M_CALL_IMM(0);                            /* passing exception pc */
2306                         M_POP(REG_ITMP2_XPC);
2307
2308                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2309                         M_JMP(REG_ITMP3);
2310                         break;
2311
2312                 case ICMD_GOTO:         /* ... ==> ...                                */
2313
2314                         M_JMP_IMM(0);
2315                         codegen_addreference(cd, iptr->dst.block);
2316                         break;
2317
2318                 case ICMD_JSR:          /* ... ==> ...                                */
2319
2320                         M_CALL_IMM(0);
2321                         codegen_addreference(cd, iptr->sx.s23.s3.jsrtarget.block);
2322                         break;
2323
2324                 case ICMD_RET:          /* ... ==> ...                                */
2325                                         /* s1.localindex = local variable             */
2326
2327                         M_JMP_IMM(0);
2328                         codegen_addreference(cd, iptr->dst.block);
2329                         break;
2330
2331                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2332
2333                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2334                         M_TEST(s1);
2335                         M_BEQ(0);
2336                         codegen_addreference(cd, iptr->dst.block);
2337                         break;
2338
2339                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2340
2341                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2342                         M_TEST(s1);
2343                         M_BNE(0);
2344                         codegen_addreference(cd, iptr->dst.block);
2345                         break;
2346
2347                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2348
2349                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2350                         M_ICMP_IMM(iptr->sx.val.i, s1);
2351                         M_BEQ(0);
2352                         codegen_addreference(cd, iptr->dst.block);
2353                         break;
2354
2355                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2356
2357                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2358                         M_ICMP_IMM(iptr->sx.val.i, s1);
2359                         M_BLT(0);
2360                         codegen_addreference(cd, iptr->dst.block);
2361                         break;
2362
2363                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2364
2365                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2366                         M_ICMP_IMM(iptr->sx.val.i, s1);
2367                         M_BLE(0);
2368                         codegen_addreference(cd, iptr->dst.block);
2369                         break;
2370
2371                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2372
2373                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2374                         M_ICMP_IMM(iptr->sx.val.i, s1);
2375                         M_BNE(0);
2376                         codegen_addreference(cd, iptr->dst.block);
2377                         break;
2378
2379                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2380
2381                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2382                         M_ICMP_IMM(iptr->sx.val.i, s1);
2383                         M_BGT(0);
2384                         codegen_addreference(cd, iptr->dst.block);
2385                         break;
2386
2387                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2388
2389                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2390                         M_ICMP_IMM(iptr->sx.val.i, s1);
2391                         M_BGE(0);
2392                         codegen_addreference(cd, iptr->dst.block);
2393                         break;
2394
2395                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2396
2397                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2398                         if (IS_IMM32(iptr->sx.val.l))
2399                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2400                         else {
2401                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2402                                 M_LCMP(REG_ITMP2, s1);
2403                         }
2404                         M_BEQ(0);
2405                         codegen_addreference(cd, iptr->dst.block);
2406                         break;
2407
2408                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2409
2410                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2411                         if (IS_IMM32(iptr->sx.val.l))
2412                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2413                         else {
2414                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2415                                 M_LCMP(REG_ITMP2, s1);
2416                         }
2417                         M_BLT(0);
2418                         codegen_addreference(cd, iptr->dst.block);
2419                         break;
2420
2421                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2422
2423                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2424                         if (IS_IMM32(iptr->sx.val.l))
2425                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2426                         else {
2427                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2428                                 M_LCMP(REG_ITMP2, s1);
2429                         }
2430                         M_BLE(0);
2431                         codegen_addreference(cd, iptr->dst.block);
2432                         break;
2433
2434                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2435
2436                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2437                         if (IS_IMM32(iptr->sx.val.l))
2438                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2439                         else {
2440                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2441                                 M_LCMP(REG_ITMP2, s1);
2442                         }
2443                         M_BNE(0);
2444                         codegen_addreference(cd, iptr->dst.block);
2445                         break;
2446
2447                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2448
2449                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2450                         if (IS_IMM32(iptr->sx.val.l))
2451                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2452                         else {
2453                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2454                                 M_LCMP(REG_ITMP2, s1);
2455                         }
2456                         M_BGT(0);
2457                         codegen_addreference(cd, iptr->dst.block);
2458                         break;
2459
2460                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2461
2462                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2463                         if (IS_IMM32(iptr->sx.val.l))
2464                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2465                         else {
2466                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2467                                 M_LCMP(REG_ITMP2, s1);
2468                         }
2469                         M_BGE(0);
2470                         codegen_addreference(cd, iptr->dst.block);
2471                         break;
2472
2473                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2474
2475                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2476                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2477                         M_ICMP(s2, s1);
2478                         M_BEQ(0);
2479                         codegen_addreference(cd, iptr->dst.block);
2480                         break;
2481
2482                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2483                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2484
2485                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2486                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2487                         M_LCMP(s2, s1);
2488                         M_BEQ(0);
2489                         codegen_addreference(cd, iptr->dst.block);
2490                         break;
2491
2492                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2493
2494                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2495                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2496                         M_ICMP(s2, s1);
2497                         M_BNE(0);
2498                         codegen_addreference(cd, iptr->dst.block);
2499                         break;
2500
2501                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2502                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2503
2504                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2505                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2506                         M_LCMP(s2, s1);
2507                         M_BNE(0);
2508                         codegen_addreference(cd, iptr->dst.block);
2509                         break;
2510
2511                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2512
2513                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2514                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2515                         M_ICMP(s2, s1);
2516                         M_BLT(0);
2517                         codegen_addreference(cd, iptr->dst.block);
2518                         break;
2519
2520                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2521
2522                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2523                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2524                         M_LCMP(s2, s1);
2525                         M_BLT(0);
2526                         codegen_addreference(cd, iptr->dst.block);
2527                         break;
2528
2529                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2530
2531                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2532                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2533                         M_ICMP(s2, s1);
2534                         M_BGT(0);
2535                         codegen_addreference(cd, iptr->dst.block);
2536                         break;
2537
2538                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2539
2540                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2541                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2542                         M_LCMP(s2, s1);
2543                         M_BGT(0);
2544                         codegen_addreference(cd, iptr->dst.block);
2545                         break;
2546
2547                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2548
2549                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2550                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2551                         M_ICMP(s2, s1);
2552                         M_BLE(0);
2553                         codegen_addreference(cd, iptr->dst.block);
2554                         break;
2555
2556                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2557
2558                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2559                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2560                         M_LCMP(s2, s1);
2561                         M_BLE(0);
2562                         codegen_addreference(cd, iptr->dst.block);
2563                         break;
2564
2565                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2566
2567                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2568                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2569                         M_ICMP(s2, s1);
2570                         M_BGE(0);
2571                         codegen_addreference(cd, iptr->dst.block);
2572                         break;
2573
2574                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2575
2576                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2577                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2578                         M_LCMP(s2, s1);
2579                         M_BGE(0);
2580                         codegen_addreference(cd, iptr->dst.block);
2581                         break;
2582
2583                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2584                 case ICMD_LRETURN:
2585
2586                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2587                         M_INTMOVE(s1, REG_RESULT);
2588                         goto nowperformreturn;
2589
2590                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2591
2592                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2593                         M_INTMOVE(s1, REG_RESULT);
2594
2595 #ifdef ENABLE_VERIFIER
2596                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2597                                 PROFILE_CYCLE_STOP;
2598
2599                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2600                                                                         iptr->sx.s23.s2.uc, 0);
2601
2602                                 if (opt_showdisassemble) {
2603                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2604                                 }
2605
2606                                 PROFILE_CYCLE_START;
2607                         }
2608 #endif /* ENABLE_VERIFIER */
2609                         goto nowperformreturn;
2610
2611                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2612                 case ICMD_DRETURN:
2613
2614                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2615                         M_FLTMOVE(s1, REG_FRESULT);
2616                         goto nowperformreturn;
2617
2618                 case ICMD_RETURN:      /* ...  ==> ...                                */
2619
2620 nowperformreturn:
2621                         {
2622                         s4 i, p;
2623
2624                         p = cd->stackframesize;
2625
2626 #if !defined(NDEBUG)
2627                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2628                                 emit_verbosecall_exit(jd);
2629 #endif /* !defined(NDEBUG) */
2630
2631 #if defined(ENABLE_THREADS)
2632                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2633                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2634
2635                                 /* we need to save the proper return value */
2636                                 switch (iptr->opc) {
2637                                 case ICMD_IRETURN:
2638                                 case ICMD_ARETURN:
2639                                 case ICMD_LRETURN:
2640                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2641                                         break;
2642                                 case ICMD_FRETURN:
2643                                 case ICMD_DRETURN:
2644                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2645                                         break;
2646                                 }
2647
2648                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2649                                 M_CALL(REG_ITMP1);
2650
2651                                 /* and now restore the proper return value */
2652                                 switch (iptr->opc) {
2653                                 case ICMD_IRETURN:
2654                                 case ICMD_ARETURN:
2655                                 case ICMD_LRETURN:
2656                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2657                                         break;
2658                                 case ICMD_FRETURN:
2659                                 case ICMD_DRETURN:
2660                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2661                                         break;
2662                                 }
2663                         }
2664 #endif
2665
2666                         /* restore saved registers */
2667
2668                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2669                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2670                         }
2671                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2672                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2673                         }
2674
2675                         /* deallocate stack */
2676
2677                         if (cd->stackframesize)
2678                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2679
2680                         /* generate method profiling code */
2681
2682                         PROFILE_CYCLE_STOP;
2683
2684                         M_RET;
2685                         }
2686                         break;
2687
2688
2689                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2690                         {
2691                                 s4 i, l;
2692                                 branch_target_t *table;
2693
2694                                 table = iptr->dst.table;
2695
2696                                 l = iptr->sx.s23.s2.tablelow;
2697                                 i = iptr->sx.s23.s3.tablehigh;
2698
2699                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2700                                 M_INTMOVE(s1, REG_ITMP1);
2701
2702                                 if (l != 0)
2703                                         M_ISUB_IMM(l, REG_ITMP1);
2704
2705                                 /* number of targets */
2706                                 i = i - l + 1;
2707
2708                 /* range check */
2709                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2710                                 M_BA(0);
2711
2712                                 codegen_addreference(cd, table[0].block); /* default target */
2713
2714                                 /* build jump table top down and use address of lowest entry */
2715
2716                                 table += i;
2717
2718                                 while (--i >= 0) {
2719                                         dseg_addtarget(cd, table->block);
2720                                         --table;
2721                                 }
2722
2723                                 /* length of dataseg after last dseg_addtarget is used
2724                                    by load */
2725
2726                                 M_MOV_IMM(0, REG_ITMP2);
2727                                 dseg_adddata(cd);
2728                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2729                                 M_JMP(REG_ITMP1);
2730                         }
2731                         break;
2732
2733
2734                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2735                         {
2736                                 s4 i;
2737                                 lookup_target_t *lookup;
2738
2739                                 lookup = iptr->dst.lookup;
2740
2741                                 i = iptr->sx.s23.s2.lookupcount;
2742
2743                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2744                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2745
2746                                 while (--i >= 0) {
2747                                         M_ICMP_IMM(lookup->value, s1);
2748                                         M_BEQ(0);
2749                                         codegen_addreference(cd, lookup->target.block);
2750                                         lookup++;
2751                                 }
2752
2753                                 M_JMP_IMM(0);
2754
2755                                 codegen_addreference(cd, iptr->sx.s23.s3.lookupdefault.block);
2756                         }
2757                         break;
2758
2759
2760                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2761
2762                         bte = iptr->sx.s23.s3.bte;
2763                         md  = bte->md;
2764                         goto gen_method;
2765
2766                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2767
2768                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2769                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2770                 case ICMD_INVOKEINTERFACE:
2771
2772                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2773                                 lm = NULL;
2774                                 um = iptr->sx.s23.s3.um;
2775                                 md = um->methodref->parseddesc.md;
2776                         }
2777                         else {
2778                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2779                                 um = NULL;
2780                                 md = lm->parseddesc;
2781                         }
2782
2783 gen_method:
2784                         s3 = md->paramcount;
2785
2786                         MCODECHECK((20 * s3) + 128);
2787
2788                         /* copy arguments to registers or stack location */
2789
2790                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2791                                 s1 = iptr->sx.s23.s2.args[s3];
2792                                 var1 = jd->var + s1;
2793
2794                                 /* Already Preallocated (ARGVAR) ? */
2795                                 if (var1->flags & PREALLOC)
2796                                         continue;
2797
2798                                 if (IS_INT_LNG_TYPE(var1->type)) {
2799                                         if (!md->params[s3].inmemory) {
2800                                                 s1 = rd->argintregs[md->params[s3].regoff];
2801                                                 d = emit_load(jd, iptr, var1, s1);
2802                                                 M_INTMOVE(d, s1);
2803                                         }
2804                                         else {
2805                                                 d = emit_load(jd, iptr, var1, REG_ITMP1);
2806                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
2807                                         }
2808                                 }
2809                                 else {
2810                                         if (!md->params[s3].inmemory) {
2811                                                 s1 = rd->argfltregs[md->params[s3].regoff];
2812                                                 d = emit_load(jd, iptr, var1, s1);
2813                                                 M_FLTMOVE(d, s1);
2814                                         }
2815                                         else {
2816                                                 d = emit_load(jd, iptr, var1, REG_FTMP1);
2817
2818                                                 if (IS_2_WORD_TYPE(var1->type))
2819                                                         M_DST(d, REG_SP, md->params[s3].regoff * 8);
2820                                                 else
2821                                                         M_FST(d, REG_SP, md->params[s3].regoff * 8);
2822                                         }
2823                                 }
2824                         }
2825
2826                         /* generate method profiling code */
2827
2828                         PROFILE_CYCLE_STOP;
2829
2830                         switch (iptr->opc) {
2831                         case ICMD_BUILTIN:
2832                                 M_MOV_IMM(bte->fp, REG_ITMP1);
2833                                 M_CALL(REG_ITMP1);
2834
2835                                 if (INSTRUCTION_MUST_CHECK(iptr)) {
2836                                         M_TEST(REG_RESULT);
2837                                         M_BEQ(0);
2838                                         codegen_add_fillinstacktrace_ref(cd);
2839                                 }
2840                                 break;
2841
2842                         case ICMD_INVOKESPECIAL:
2843                                 M_TEST(REG_A0);
2844                                 M_BEQ(0);
2845                                 codegen_add_nullpointerexception_ref(cd);
2846
2847                                 /* fall through */
2848
2849                         case ICMD_INVOKESTATIC:
2850                                 if (lm == NULL) {
2851                                         disp = dseg_addaddress(cd, NULL);
2852                                         disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2853
2854                                         /* must be calculated before codegen_addpatchref */
2855
2856                                         if (opt_showdisassemble)
2857                                                 disp -= PATCHER_CALL_SIZE;
2858
2859                                         codegen_addpatchref(cd, PATCHER_invokestatic_special,
2860                                                                                 um, disp);
2861
2862                                         if (opt_showdisassemble) {
2863                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2864                                         }
2865
2866 /*                                      a = 0; */
2867                                 }
2868                                 else {
2869                                         disp = dseg_addaddress(cd, lm->stubroutine);
2870                                         disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2871
2872 /*                                      a = (ptrint) lm->stubroutine; */
2873                                 }
2874
2875 /*                              M_MOV_IMM(a, REG_ITMP2); */
2876                                 M_ALD(REG_ITMP2, RIP, disp);
2877                                 M_CALL(REG_ITMP2);
2878                                 break;
2879
2880                         case ICMD_INVOKEVIRTUAL:
2881                                 gen_nullptr_check(REG_A0);
2882
2883                                 if (lm == NULL) {
2884                                         codegen_addpatchref(cd, PATCHER_invokevirtual, um, 0);
2885
2886                                         if (opt_showdisassemble) {
2887                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2888                                         }
2889
2890                                         s1 = 0;
2891                                 }
2892                                 else
2893                                         s1 = OFFSET(vftbl_t, table[0]) +
2894                                                 sizeof(methodptr) * lm->vftblindex;
2895
2896                                 M_ALD(REG_METHODPTR, REG_A0,
2897                                           OFFSET(java_objectheader, vftbl));
2898                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2899                                 M_CALL(REG_ITMP3);
2900                                 break;
2901
2902                         case ICMD_INVOKEINTERFACE:
2903                                 gen_nullptr_check(REG_A0);
2904
2905                                 if (lm == NULL) {
2906                                         codegen_addpatchref(cd, PATCHER_invokeinterface, um, 0);
2907
2908                                         if (opt_showdisassemble) {
2909                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2910                                         }
2911
2912                                         s1 = 0;
2913                                         s2 = 0;
2914                                 }
2915                                 else {
2916                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2917                                                 sizeof(methodptr) * lm->class->index;
2918
2919                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2920                                 }
2921
2922                                 M_ALD(REG_METHODPTR, REG_A0,
2923                                           OFFSET(java_objectheader, vftbl));
2924                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2925                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2926                                 M_CALL(REG_ITMP3);
2927                                 break;
2928                         }
2929
2930                         /* generate method profiling code */
2931
2932                         PROFILE_CYCLE_START;
2933
2934                         /* store return value */
2935
2936                         d = md->returntype.type;
2937
2938                         if (d != TYPE_VOID) {
2939                                 if (IS_INT_LNG_TYPE(d)) {
2940                                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2941                                         M_INTMOVE(REG_RESULT, s1);
2942                                 }
2943                                 else {
2944                                         s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2945                                         M_FLTMOVE(REG_FRESULT, s1);
2946                                 }
2947                                 emit_store_dst(jd, iptr, s1);
2948                         }
2949                         break;
2950
2951
2952                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2953
2954                                       /* val.a: (classinfo *) superclass              */
2955
2956                         /*  superclass is an interface:
2957                          *
2958                          *  OK if ((sub == NULL) ||
2959                          *         (sub->vftbl->interfacetablelength > super->index) &&
2960                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
2961                          *
2962                          *  superclass is a class:
2963                          *
2964                          *  OK if ((sub == NULL) || (0
2965                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
2966                          *         super->vftbl->diffval));
2967                          */
2968
2969                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2970                                 /* object type cast-check */
2971
2972                                 classinfo *super;
2973                                 vftbl_t   *supervftbl;
2974                                 s4         superindex;
2975
2976                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2977                                         super = NULL;
2978                                         superindex = 0;
2979                                         supervftbl = NULL;
2980                                 }
2981                                 else {
2982                                         super = iptr->sx.s23.s3.c.cls;
2983                                         superindex = super->index;
2984                                         supervftbl = super->vftbl;
2985                                 }
2986
2987 #if defined(ENABLE_THREADS)
2988                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
2989 #endif
2990                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2991
2992                                 /* calculate interface checkcast code size */
2993
2994                                 s2 = 3; /* mov_membase_reg */
2995                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
2996
2997                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
2998                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
2999                                         3 /* test */ + 6 /* jcc */;
3000
3001                                 if (super == NULL)
3002                                         s2 += (opt_showdisassemble ? 5 : 0);
3003
3004                                 /* calculate class checkcast code size */
3005
3006                                 s3 = 3; /* mov_membase_reg */
3007                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3008                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3009
3010 #if 0
3011                                 if (s1 != REG_ITMP1) {
3012                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3013                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3014                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3015                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3016                                         a += 3;    /* sub */
3017
3018                                 } else
3019 #endif
3020                                         {
3021                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3022                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3023                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3024                                         }
3025
3026                                 s3 += 3 /* cmp */ + 6 /* jcc */;
3027
3028                                 if (super == NULL)
3029                                         s3 += (opt_showdisassemble ? 5 : 0);
3030
3031                                 /* if class is not resolved, check which code to call */
3032
3033                                 if (super == NULL) {
3034                                         M_TEST(s1);
3035                                         M_BEQ(6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3036
3037                                         codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3038                                                                                 iptr->sx.s23.s3.c.ref, 0);
3039
3040                                         if (opt_showdisassemble) {
3041                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3042                                         }
3043
3044                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
3045                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
3046                                         M_BEQ(s2 + 5);
3047                                 }
3048
3049                                 /* interface checkcast code */
3050
3051                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
3052                                         if (super != NULL) {
3053                                                 M_TEST(s1);
3054                                                 M_BEQ(s2);
3055                                         }
3056
3057                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3058
3059                                         if (super == NULL) {
3060                                                 codegen_addpatchref(cd,
3061                                                                                         PATCHER_checkcast_instanceof_interface,
3062                                                                                         iptr->sx.s23.s3.c.ref,
3063                                                                                         0);
3064
3065                                                 if (opt_showdisassemble) {
3066                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3067                                                 }
3068                                         }
3069
3070                                         emit_movl_membase32_reg(cd, REG_ITMP2,
3071                                                                                           OFFSET(vftbl_t, interfacetablelength),
3072                                                                                           REG_ITMP3);
3073                                         /* XXX TWISTI: should this be int arithmetic? */
3074                                         M_LSUB_IMM32(superindex, REG_ITMP3);
3075                                         M_TEST(REG_ITMP3);
3076                                         M_BLE(0);
3077                                         codegen_add_classcastexception_ref(cd, s1);
3078                                         emit_mov_membase32_reg(cd, REG_ITMP2,
3079                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
3080                                                                                          superindex * sizeof(methodptr*),
3081                                                                                          REG_ITMP3);
3082                                         M_TEST(REG_ITMP3);
3083                                         M_BEQ(0);
3084                                         codegen_add_classcastexception_ref(cd, s1);
3085
3086                                         if (super == NULL)
3087                                                 M_JMP_IMM(s3);
3088                                 }
3089
3090                                 /* class checkcast code */
3091
3092                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
3093                                         if (super != NULL) {
3094                                                 M_TEST(s1);
3095                                                 M_BEQ(s3);
3096                                         }
3097
3098                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3099
3100                                         if (super == NULL) {
3101                                                 codegen_addpatchref(cd, PATCHER_checkcast_class,
3102                                                                                         iptr->sx.s23.s3.c.ref,
3103                                                                                         0);
3104
3105                                                 if (opt_showdisassemble) {
3106                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3107                                                 }
3108                                         }
3109
3110                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3111 #if defined(ENABLE_THREADS)
3112                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3113 #endif
3114                                         emit_movl_membase32_reg(cd, REG_ITMP2,
3115                                                                                           OFFSET(vftbl_t, baseval),
3116                                                                                           REG_ITMP2);
3117                                         /*                                      if (s1 != REG_ITMP1) { */
3118                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3119                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
3120                                         /*                                                                                              REG_ITMP1); */
3121                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3122                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3123                                         /*                                                                                              REG_ITMP3); */
3124                                         /*  #if defined(ENABLE_THREADS) */
3125                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3126                                         /*  #endif */
3127                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
3128
3129                                         /*                                      } else { */
3130                                         emit_movl_membase32_reg(cd, REG_ITMP3,
3131                                                                                           OFFSET(vftbl_t, baseval),
3132                                                                                           REG_ITMP3);
3133                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3134                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3135                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3136                                         /*                                      } */
3137 #if defined(ENABLE_THREADS)
3138                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3139 #endif
3140                                         M_LCMP(REG_ITMP3, REG_ITMP2);
3141                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3142                                         codegen_add_classcastexception_ref(cd, s1);
3143                                 }
3144
3145                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
3146                         }
3147                         else {
3148                                 /* array type cast-check */
3149
3150                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
3151                                 M_INTMOVE(s1, REG_A0);
3152
3153                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3154                                         codegen_addpatchref(cd, PATCHER_builtin_arraycheckcast,
3155                                                                                 iptr->sx.s23.s3.c.ref, 0);
3156
3157                                         if (opt_showdisassemble) {
3158                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3159                                         }
3160                                 }
3161
3162                                 M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
3163                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
3164                                 M_CALL(REG_ITMP1);
3165
3166                                 /* s1 may have been destroyed over the function call */
3167                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
3168                                 M_TEST(REG_RESULT);
3169                                 M_BEQ(0);
3170                                 codegen_add_classcastexception_ref(cd, s1);
3171
3172                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
3173                         }
3174
3175                         M_INTMOVE(s1, d);
3176                         emit_store_dst(jd, iptr, d);
3177                         break;
3178
3179                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3180
3181                                       /* val.a: (classinfo *) superclass              */
3182
3183                         /*  superclass is an interface:
3184                          *
3185                          *  return (sub != NULL) &&
3186                          *         (sub->vftbl->interfacetablelength > super->index) &&
3187                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3188                          *
3189                          *  superclass is a class:
3190                          *
3191                          *  return ((sub != NULL) && (0
3192                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3193                          *          super->vftbl->diffvall));
3194                          */
3195
3196                         {
3197                         classinfo *super;
3198                         vftbl_t   *supervftbl;
3199                         s4         superindex;
3200
3201                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3202                                 super = NULL;
3203                                 superindex = 0;
3204                                 supervftbl = NULL;
3205
3206                         } else {
3207                                 super = iptr->sx.s23.s3.c.cls;
3208                                 superindex = super->index;
3209                                 supervftbl = super->vftbl;
3210                         }
3211
3212 #if defined(ENABLE_THREADS)
3213             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3214 #endif
3215
3216                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
3217                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
3218                         if (s1 == d) {
3219                                 M_INTMOVE(s1, REG_ITMP1);
3220                                 s1 = REG_ITMP1;
3221                         }
3222
3223                         /* calculate interface instanceof code size */
3224
3225                         s2 = 3; /* mov_membase_reg */
3226                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3227                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3228                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3229                                 3 /* test */ + 4 /* setcc */;
3230
3231                         if (!super)
3232                                 s2 += (opt_showdisassemble ? 5 : 0);
3233
3234                         /* calculate class instanceof code size */
3235
3236                         s3 = 3; /* mov_membase_reg */
3237                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3238                         s3 += 10; /* mov_imm_reg */
3239                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3240                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3241                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3242                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3243                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3244                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3245                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3246
3247                         if (!super)
3248                                 s3 += (opt_showdisassemble ? 5 : 0);
3249
3250                         emit_alu_reg_reg(cd, ALU_XOR, d, d);
3251
3252                         /* if class is not resolved, check which code to call */
3253
3254                         if (!super) {
3255                                 emit_test_reg_reg(cd, s1, s1);
3256                                 emit_jcc(cd, CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3257                                                                                          7 + 6 + s2 + 5 + s3));
3258
3259                                 codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3260                                                                         iptr->sx.s23.s3.c.ref, 0);
3261
3262                                 if (opt_showdisassemble) {
3263                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3264                                 }
3265
3266                                 emit_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3267                                 emit_alul_imm_reg(cd, ALU_AND, ACC_INTERFACE, REG_ITMP3);
3268                                 emit_jcc(cd, CC_Z, s2 + 5);
3269                         }
3270
3271                         /* interface instanceof code */
3272
3273                         if (!super || (super->flags & ACC_INTERFACE)) {
3274                                 if (super) {
3275                                         emit_test_reg_reg(cd, s1, s1);
3276                                         emit_jcc(cd, CC_Z, s2);
3277                                 }
3278
3279                                 emit_mov_membase_reg(cd, s1,
3280                                                                            OFFSET(java_objectheader, vftbl),
3281                                                                            REG_ITMP1);
3282                                 if (!super) {
3283                                         codegen_addpatchref(cd,
3284                                                                                 PATCHER_checkcast_instanceof_interface,
3285                                                                                 iptr->sx.s23.s3.c.ref, 0);
3286
3287                                         if (opt_showdisassemble) {
3288                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3289                                         }
3290                                 }
3291
3292                                 emit_movl_membase32_reg(cd, REG_ITMP1,
3293                                                                                   OFFSET(vftbl_t, interfacetablelength),
3294                                                                                   REG_ITMP3);
3295                                 emit_alu_imm32_reg(cd, ALU_SUB, superindex, REG_ITMP3);
3296                                 emit_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3297
3298                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3299
3300                                 emit_jcc(cd, CC_LE, a);
3301                                 emit_mov_membase32_reg(cd, REG_ITMP1,
3302                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3303                                                                                  superindex * sizeof(methodptr*),
3304                                                                                  REG_ITMP1);
3305                                 emit_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3306                                 emit_setcc_reg(cd, CC_NE, d);
3307
3308                                 if (!super)
3309                                         emit_jmp_imm(cd, s3);
3310                         }
3311
3312                         /* class instanceof code */
3313
3314                         if (!super || !(super->flags & ACC_INTERFACE)) {
3315                                 if (super) {
3316                                         emit_test_reg_reg(cd, s1, s1);
3317                                         emit_jcc(cd, CC_E, s3);
3318                                 }
3319
3320                                 emit_mov_membase_reg(cd, s1,
3321                                                                            OFFSET(java_objectheader, vftbl),
3322                                                                            REG_ITMP1);
3323
3324                                 if (!super) {
3325                                         codegen_addpatchref(cd, PATCHER_instanceof_class,
3326                                                                                 iptr->sx.s23.s3.c.ref, 0);
3327
3328                                         if (opt_showdisassemble) {
3329                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3330                                         }
3331                                 }
3332
3333                                 emit_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3334 #if defined(ENABLE_THREADS)
3335                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3336 #endif
3337                                 emit_movl_membase_reg(cd, REG_ITMP1,
3338                                                                                 OFFSET(vftbl_t, baseval),
3339                                                                                 REG_ITMP1);
3340                                 emit_movl_membase_reg(cd, REG_ITMP2,
3341                                                                                 OFFSET(vftbl_t, diffval),
3342                                                                                 REG_ITMP3);
3343                                 emit_movl_membase_reg(cd, REG_ITMP2,
3344                                                                                 OFFSET(vftbl_t, baseval),
3345                                                                                 REG_ITMP2);
3346 #if defined(ENABLE_THREADS)
3347                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3348 #endif
3349                                 emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
3350                                 emit_alu_reg_reg(cd, ALU_XOR, d, d); /* may be REG_ITMP2 */
3351                                 emit_alu_reg_reg(cd, ALU_CMP, REG_ITMP3, REG_ITMP1);
3352                                 emit_setcc_reg(cd, CC_BE, d);
3353                         }
3354                         emit_store_dst(jd, iptr, d);
3355                         }
3356                         break;
3357
3358                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3359
3360                         /* check for negative sizes and copy sizes to stack if necessary  */
3361
3362                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
3363
3364                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
3365
3366                                 /* copy SAVEDVAR sizes to stack */
3367                                 s3 = iptr->sx.s23.s2.args[s1];
3368                                 var1 = jd->var + s3;
3369
3370                                 /* Already Preallocated (ARGVAR) ? */
3371                                 if (!(var1->flags & PREALLOC)) {
3372                                         s2 = emit_load(jd, iptr, var1, REG_ITMP1);
3373                                         M_LST(s2, REG_SP, s1 * 8);
3374                                 }
3375                         }
3376
3377                         /* is a patcher function set? */
3378
3379                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3380                                 codegen_addpatchref(cd, PATCHER_builtin_multianewarray,
3381                                                                         iptr->sx.s23.s3.c.ref, 0);
3382
3383                                 if (opt_showdisassemble) {
3384                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3385                                 }
3386                         }
3387
3388                         /* a0 = dimension count */
3389
3390                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
3391
3392                         /* a1 = classinfo */
3393
3394                         M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
3395
3396                         /* a2 = pointer to dimensions = stack pointer */
3397
3398                         M_MOV(REG_SP, REG_A2);
3399
3400                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3401                         M_CALL(REG_ITMP1);
3402
3403                         /* check for exception before result assignment */
3404
3405                         M_TEST(REG_RESULT);
3406                         M_BEQ(0);
3407                         codegen_add_fillinstacktrace_ref(cd);
3408
3409                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
3410                         M_INTMOVE(REG_RESULT, s1);
3411                         emit_store_dst(jd, iptr, s1);
3412                         break;
3413
3414                 default:
3415                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3416                         return false;
3417         } /* switch */
3418
3419         } /* for instruction */
3420
3421         /* copy values to interface registers */
3422
3423         len = bptr->outdepth;
3424         MCODECHECK(512);
3425
3426         /* At the end of a basic block we may have to append some nops,
3427            because the patcher stub calling code might be longer than the
3428            actual instruction. So codepatching does not change the
3429            following block unintentionally. */
3430
3431         if (cd->mcodeptr < cd->lastmcodeptr) {
3432                 while (cd->mcodeptr < cd->lastmcodeptr) {
3433                         M_NOP;
3434                 }
3435         }
3436
3437         } /* if (bptr -> flags >= BBREACHED) */
3438         } /* for basic block */
3439
3440         dseg_createlinenumbertable(cd);
3441
3442         /* generate stubs */
3443
3444         emit_exception_stubs(jd);
3445         emit_patcher_stubs(jd);
3446 #if 0
3447         emit_replacement_stubs(jd);
3448 #endif
3449
3450         codegen_finish(jd);
3451
3452         /* everything's ok */
3453
3454         return true;
3455 }
3456
3457
3458 /* createcompilerstub **********************************************************
3459
3460    Creates a stub routine which calls the compiler.
3461
3462 *******************************************************************************/
3463
3464 #define COMPILERSTUB_DATASIZE    3 * SIZEOF_VOID_P
3465 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
3466
3467 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
3468
3469
3470 u1 *createcompilerstub(methodinfo *m)
3471 {
3472         u1          *s;                     /* memory to hold the stub            */
3473         ptrint      *d;
3474         codeinfo    *code;
3475         codegendata *cd;
3476         s4           dumpsize;
3477
3478         s = CNEW(u1, COMPILERSTUB_SIZE);
3479
3480         /* set data pointer and code pointer */
3481
3482         d = (ptrint *) s;
3483         s = s + COMPILERSTUB_DATASIZE;
3484
3485         /* mark start of dump memory area */
3486
3487         dumpsize = dump_size();
3488
3489         cd = DNEW(codegendata);
3490         cd->mcodeptr = s;
3491
3492         /* Store the codeinfo pointer in the same place as in the
3493            methodheader for compiled methods. */
3494
3495         code = code_codeinfo_new(m);
3496
3497         d[0] = (ptrint) asm_call_jit_compiler;
3498         d[1] = (ptrint) m;
3499         d[2] = (ptrint) code;
3500
3501         /* code for the stub */
3502
3503         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
3504         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
3505         M_JMP(REG_ITMP3);
3506
3507 #if defined(ENABLE_STATISTICS)
3508         if (opt_stat)
3509                 count_cstub_len += COMPILERSTUB_SIZE;
3510 #endif
3511
3512         /* release dump area */
3513
3514         dump_release(dumpsize);
3515
3516         return s;
3517 }
3518
3519
3520 /* createnativestub ************************************************************
3521
3522    Creates a stub routine which calls a native method.
3523
3524 *******************************************************************************/
3525
3526 u1 *createnativestub(functionptr f, jitdata *jd, methoddesc *nmd)
3527 {
3528         methodinfo   *m;
3529         codeinfo     *code;
3530         codegendata  *cd;
3531         registerdata *rd;
3532         methoddesc   *md;
3533         s4            nativeparams;
3534         s4            i, j;                 /* count variables                    */
3535         s4            t;
3536         s4            s1, s2;
3537
3538         /* get required compiler data */
3539
3540         m    = jd->m;
3541         code = jd->code;
3542         cd   = jd->cd;
3543         rd   = jd->rd;
3544
3545         /* initialize variables */
3546
3547         md = m->parseddesc;
3548         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
3549
3550         /* calculate stack frame size */
3551
3552         cd->stackframesize =
3553                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
3554                 sizeof(localref_table) / SIZEOF_VOID_P +
3555                 INT_ARG_CNT + FLT_ARG_CNT +
3556                 1 +                       /* functionptr, TODO: store in data segment */
3557                 nmd->memuse;
3558
3559         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
3560
3561         /* create method header */
3562
3563         (void) dseg_addaddress(cd, code);                      /* CodeinfoPointer */
3564         (void) dseg_adds4(cd, cd->stackframesize * 8);         /* FrameSize       */
3565         (void) dseg_adds4(cd, 0);                              /* IsSync          */
3566         (void) dseg_adds4(cd, 0);                              /* IsLeaf          */
3567         (void) dseg_adds4(cd, 0);                              /* IntSave         */
3568         (void) dseg_adds4(cd, 0);                              /* FltSave         */
3569         (void) dseg_addlinenumbertablesize(cd);
3570         (void) dseg_adds4(cd, 0);                              /* ExTableSize     */
3571
3572         /* generate native method profiling code */
3573
3574         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3575                 /* count frequency */
3576
3577                 M_MOV_IMM(code, REG_ITMP3);
3578                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3579         }
3580
3581         /* generate stub code */
3582
3583         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3584
3585 #if !defined(NDEBUG)
3586         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3587                 emit_verbosecall_enter(jd);
3588 #endif
3589
3590         /* get function address (this must happen before the stackframeinfo) */
3591
3592 #if !defined(WITH_STATIC_CLASSPATH)
3593         if (f == NULL) {
3594                 codegen_addpatchref(cd, PATCHER_resolve_native, m, 0);
3595
3596                 if (opt_showdisassemble) {
3597                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3598                 }
3599         }
3600 #endif
3601
3602         M_MOV_IMM(f, REG_ITMP3);
3603
3604
3605         /* save integer and float argument registers */
3606
3607         for (i = 0, j = 0; i < md->paramcount; i++) {
3608                 if (!md->params[i].inmemory) {
3609                         s1 = md->params[i].regoff;
3610
3611                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3612                                 M_LST(rd->argintregs[s1], REG_SP, j * 8);
3613                         else
3614                                 M_DST(rd->argfltregs[s1], REG_SP, j * 8);
3615
3616                         j++;
3617                 }
3618         }
3619
3620         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3621
3622         /* create dynamic stack info */
3623
3624         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3625         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3626         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3627         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3628         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3629         M_CALL(REG_ITMP1);
3630
3631         /* restore integer and float argument registers */
3632
3633         for (i = 0, j = 0; i < md->paramcount; i++) {
3634                 if (!md->params[i].inmemory) {
3635                         s1 = md->params[i].regoff;
3636
3637                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3638                                 M_LLD(rd->argintregs[s1], REG_SP, j * 8);
3639                         else
3640                                 M_DLD(rd->argfltregs[s1], REG_SP, j * 8);
3641
3642                         j++;
3643                 }
3644         }
3645
3646         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3647
3648
3649         /* copy or spill arguments to new locations */
3650
3651         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3652                 t = md->paramtypes[i].type;
3653
3654                 if (IS_INT_LNG_TYPE(t)) {
3655                         if (!md->params[i].inmemory) {
3656                                 s1 = rd->argintregs[md->params[i].regoff];
3657
3658                                 if (!nmd->params[j].inmemory) {
3659                                         s2 = rd->argintregs[nmd->params[j].regoff];
3660                                         M_INTMOVE(s1, s2);
3661
3662                                 } else {
3663                                         s2 = nmd->params[j].regoff;
3664                                         M_LST(s1, REG_SP, s2 * 8);
3665                                 }
3666
3667                         } else {
3668                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3669                                 s2 = nmd->params[j].regoff;
3670                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
3671                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
3672                         }
3673
3674                 } else {
3675                         /* We only copy spilled float arguments, as the float argument    */
3676                         /* registers keep unchanged.                                      */
3677
3678                         if (md->params[i].inmemory) {
3679                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3680                                 s2 = nmd->params[j].regoff;
3681
3682                                 if (IS_2_WORD_TYPE(t)) {
3683                                         M_DLD(REG_FTMP1, REG_SP, s1 * 8);
3684                                         M_DST(REG_FTMP1, REG_SP, s2 * 8);
3685                                 } else {
3686                                         M_FLD(REG_FTMP1, REG_SP, s1 * 8);
3687                                         M_FST(REG_FTMP1, REG_SP, s2 * 8);
3688                                 }
3689                         }
3690                 }
3691         }
3692
3693         /* put class into second argument register */
3694
3695         if (m->flags & ACC_STATIC)
3696                 M_MOV_IMM(m->class, REG_A1);
3697
3698         /* put env into first argument register */
3699
3700         M_MOV_IMM(_Jv_env, REG_A0);
3701
3702         /* do the native function call */
3703
3704         M_CALL(REG_ITMP3);
3705
3706         /* save return value */
3707
3708         if (md->returntype.type != TYPE_VOID) {
3709                 if (IS_INT_LNG_TYPE(md->returntype.type))
3710                         M_LST(REG_RESULT, REG_SP, 0 * 8);
3711                 else
3712                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
3713         }
3714
3715 #if !defined(NDEBUG)
3716         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3717                 emit_verbosecall_exit(jd);
3718 #endif
3719
3720         /* remove native stackframe info */
3721
3722         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3723         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3724         M_CALL(REG_ITMP1);
3725         M_MOV(REG_RESULT, REG_ITMP3);
3726
3727         /* restore return value */
3728
3729         if (md->returntype.type != TYPE_VOID) {
3730                 if (IS_INT_LNG_TYPE(md->returntype.type))
3731                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
3732                 else
3733                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3734         }
3735
3736         /* remove stackframe */
3737
3738         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3739
3740         /* test for exception */
3741
3742         M_TEST(REG_ITMP3);
3743         M_BNE(1);
3744         M_RET;
3745
3746         /* handle exception */
3747
3748         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3749         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3750         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3751
3752         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3753         M_JMP(REG_ITMP3);
3754
3755         /* generate patcher stubs */
3756
3757         emit_patcher_stubs(jd);
3758
3759         codegen_finish(jd);
3760
3761         return code->entrypoint;
3762 }
3763
3764
3765 /*
3766  * These are local overrides for various environment variables in Emacs.
3767  * Please do not remove this and leave it at the end of the file, where
3768  * Emacs will automagically detect them.
3769  * ---------------------------------------------------------------------
3770  * Local variables:
3771  * mode: c
3772  * indent-tabs-mode: t
3773  * c-basic-offset: 4
3774  * tab-width: 4
3775  * End:
3776  * vim:noexpandtab:sw=4:ts=4:
3777  */