src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29             Christian Ullrich
  30             Edwin Steiner
  31
  32    $Id: codegen.c 5945 2006-11-10 16:41:12Z twisti $
  33
  34 */
  35
  36
  37 #include "config.h"
  38
  39 #include <assert.h>
  40 #include <stdio.h>
  41
  42 #include "vm/types.h"
  43
  44 #include "md-abi.h"
  45
  46 #include "vm/jit/x86_64/arch.h"
  47 #include "vm/jit/x86_64/codegen.h"
  48 #include "vm/jit/x86_64/emit.h"
  49
  50 #include "mm/memory.h"
  51 #include "native/jni.h"
  52 #include "native/native.h"
  53
  54 #if defined(ENABLE_THREADS)
  55 # include "threads/native/lock.h"
  56 #endif
  57
  58 #include "vm/builtin.h"
  59 #include "vm/exceptions.h"
  60 #include "vm/global.h"
  61 #include "vm/loader.h"
  62 #include "vm/options.h"
  63 #include "vm/statistics.h"
  64 #include "vm/stringlocal.h"
  65 #include "vm/vm.h"
  66 #include "vm/jit/asmpart.h"
  67 #include "vm/jit/codegen-common.h"
  68 #include "vm/jit/dseg.h"
  69 #include "vm/jit/emit-common.h"
  70 #include "vm/jit/jit.h"
  71 #include "vm/jit/methodheader.h"
  72 #include "vm/jit/parse.h"
  73 #include "vm/jit/patcher.h"
  74 #include "vm/jit/reg.h"
  75 #include "vm/jit/replace.h"
  76
  77 #if defined(ENABLE_LSRA)
  78 # include "vm/jit/allocator/lsra.h"
  79 #endif
  80
  81
  82 /* codegen *********************************************************************
  83
  84    Generates machine code.
  85
  86 *******************************************************************************/
  87
  88 bool codegen(jitdata *jd)
  89 {
  90         methodinfo         *m;
  91         codeinfo           *code;
  92         codegendata        *cd;
  93         registerdata       *rd;
  94         s4                  len, s1, s2, s3, d, disp;
  95         u2                  currentline;
  96         ptrint              a;
  97         varinfo            *var, *var1, *var2, *dst;
  98         basicblock         *bptr;
  99         instruction        *iptr;
 100         exception_entry    *ex;
 101         constant_classref  *cr;
 102         unresolved_class   *uc;
 103         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 104         unresolved_method  *um;
 105         builtintable_entry *bte;
 106         methoddesc         *md;
 107         fieldinfo          *fi;
 108         unresolved_field   *uf;
 109         s4                  fieldtype;
 110         rplpoint           *replacementpoint;
 111         s4                 varindex;
 112
 113         /* get required compiler data */
 114
 115         m    = jd->m;
 116         code = jd->code;
 117         cd   = jd->cd;
 118         rd   = jd->rd;
 119
 120         /* prevent compiler warnings */
 121
 122         d   = 0;
 123         lm  = NULL;
 124         um  = NULL;
 125         bte = NULL;
 126
 127         {
 128         s4 i, p, t, l;
 129         s4 savedregs_num;
 130
 131         savedregs_num = 0;
 132
 133         /* space to save used callee saved registers */
 134
 135         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 136         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 137
 138         cd->stackframesize = rd->memuse + savedregs_num;
 139
 140 #if defined(ENABLE_THREADS)
 141         /* space to save argument of monitor_enter */
 142
 143         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 144                 cd->stackframesize++;
 145 #endif
 146
 147         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 148            native code e.g. libc or jni (alignment problems with
 149            movaps). */
 150
 151         if (!jd->isleafmethod || opt_verbosecall)
 152                 cd->stackframesize |= 0x1;
 153
 154         /* create method header */
 155
 156         (void) dseg_addaddress(cd, code);                      /* CodeinfoPointer */
 157         (void) dseg_adds4(cd, cd->stackframesize * 8);         /* FrameSize       */
 158
 159 #if defined(ENABLE_THREADS)
 160         /* IsSync contains the offset relative to the stack pointer for the
 161            argument of monitor_exit used in the exception handler. Since the
 162            offset could be zero and give a wrong meaning of the flag it is
 163            offset by one.
 164         */
 165
 166         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 167                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);       /* IsSync          */
 168         else
 169 #endif
 170                 (void) dseg_adds4(cd, 0);                          /* IsSync          */
 171
 172         (void) dseg_adds4(cd, jd->isleafmethod);               /* IsLeaf          */
 173         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave         */
 174         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave         */
 175
 176         (void) dseg_addlinenumbertablesize(cd);
 177
 178         (void) dseg_adds4(cd, jd->exceptiontablelength);       /* ExTableSize     */
 179
 180         /* create exception table */
 181
 182         for (ex = jd->exceptiontable; ex != NULL; ex = ex->down) {
 183                 dseg_addtarget(cd, ex->start);
 184                 dseg_addtarget(cd, ex->end);
 185                 dseg_addtarget(cd, ex->handler);
 186                 (void) dseg_addaddress(cd, ex->catchtype.any);
 187         }
 188
 189         /* generate method profiling code */
 190
 191         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 192                 /* count frequency */
 193
 194                 M_MOV_IMM(code, REG_ITMP3);
 195                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 196
 197                 PROFILE_CYCLE_START;
 198         }
 199
 200         /* create stack frame (if necessary) */
 201
 202         if (cd->stackframesize)
 203                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 204
 205         /* save used callee saved registers */
 206
 207         p = cd->stackframesize;
 208         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 209                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 210         }
 211         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 212                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 213         }
 214
 215         /* take arguments out of register or stack frame */
 216
 217         md = m->parseddesc;
 218
 219         for (p = 0, l = 0; p < md->paramcount; p++) {
 220                 t = md->paramtypes[p].type;
 221
 222                 varindex = jd->local_map[l * 5 + t];
 223
 224                 l++;
 225                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 226                         l++;
 227
 228                 if (varindex == UNUSED)
 229                         continue;
 230
 231                 var = VAR(varindex);
 232
 233                 s1 = md->params[p].regoff;
 234
 235                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 236                         s2 = rd->argintregs[s1];
 237                         if (!md->params[p].inmemory) {           /* register arguments    */
 238                                 if (!IS_INMEMORY(var->flags)) {      /* reg arg -> register   */
 239                                         M_INTMOVE(s2, var->vv.regoff);
 240
 241                                 } else {                             /* reg arg -> spilled    */
 242                                     M_LST(s2, REG_SP, var->vv.regoff * 8);
 243                                 }
 244
 245                         } else {                                 /* stack arguments       */
 246                                 if (!IS_INMEMORY(var->flags)) {      /* stack arg -> register */
 247                                         /* + 8 for return address */
 248                                         M_LLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 249
 250                                 } else {                             /* stack arg -> spilled  */
 251                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 252                                 }
 253                         }
 254
 255                 } else {                                     /* floating args         */
 256                         if (!md->params[p].inmemory) {           /* register arguments    */
 257                                 s2 = rd->argfltregs[s1];
 258                                 if (!IS_INMEMORY(var->flags)) {      /* reg arg -> register   */
 259                                         M_FLTMOVE(s2, var->vv.regoff);
 260
 261                                 } else {                                         /* reg arg -> spilled    */
 262                                         M_DST(s2, REG_SP, var->vv.regoff * 8);
 263                                 }
 264
 265                         } else {                                 /* stack arguments       */
 266                                 if (!IS_INMEMORY(var->flags)) {      /* stack-arg -> register */
 267                                         M_DLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 268
 269                                 } else {
 270                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 271                                 }
 272                         }
 273                 }
 274         }  /* end for */
 275
 276         /* save monitorenter argument */
 277
 278 #if defined(ENABLE_THREADS)
 279         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 280                 /* stack offset for monitor argument */
 281
 282                 s1 = rd->memuse;
 283
 284                 if (opt_verbosecall) {
 285                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 286
 287                         for (p = 0; p < INT_ARG_CNT; p++)
 288                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 289
 290                         for (p = 0; p < FLT_ARG_CNT; p++)
 291                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 292
 293                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 294                 }
 295
 296                 /* decide which monitor enter function to call */
 297
 298                 if (m->flags & ACC_STATIC) {
 299                         M_MOV_IMM(&m->class->object.header, REG_A0);
 300                 }
 301                 else {
 302                         M_TEST(REG_A0);
 303                         M_BEQ(0);
 304                         codegen_add_nullpointerexception_ref(cd);
 305                 }
 306
 307                 M_AST(REG_A0, REG_SP, s1 * 8);
 308                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 309                 M_CALL(REG_ITMP1);
 310
 311                 if (opt_verbosecall) {
 312                         for (p = 0; p < INT_ARG_CNT; p++)
 313                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 314
 315                         for (p = 0; p < FLT_ARG_CNT; p++)
 316                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 317
 318                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 319                 }
 320         }
 321 #endif
 322
 323 #if !defined(NDEBUG)
 324         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 325                 emit_verbosecall_enter(jd);
 326 #endif /* !defined(NDEBUG) */
 327
 328         }
 329
 330         /* end of header generation */
 331
 332         replacementpoint = jd->code->rplpoints;
 333
 334         /* walk through all basic blocks */
 335
 336         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 337
 338                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 339
 340                 if (bptr->flags >= BBREACHED) {
 341
 342                 /* branch resolving */
 343
 344                 codegen_resolve_branchrefs(cd, bptr);
 345
 346                 /* handle replacement points */
 347
 348 #if 0
 349                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 350                         replacementpoint->pc = (u1*)(ptrint)bptr->mpc; /* will be resolved later */
 351
 352                         replacementpoint++;
 353
 354                         assert(cd->lastmcodeptr <= cd->mcodeptr);
 355                         cd->lastmcodeptr = cd->mcodeptr + 5; /* 5 byte jmp patch */
 356                 }
 357 #endif
 358
 359                 /* copy interface registers to their destination */
 360
 361                 len = bptr->indepth;
 362                 MCODECHECK(512);
 363
 364                 /* generate basicblock profiling code */
 365
 366                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 367                         /* count frequency */
 368
 369                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 370                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 371
 372                         /* if this is an exception handler, start profiling again */
 373
 374                         if (bptr->type == BBTYPE_EXH)
 375                                 PROFILE_CYCLE_START;
 376                 }
 377
 378 #if defined(ENABLE_LSRA)
 379                 if (opt_lsra) {
 380                         while (len) {
 381                                 len--;
 382                                 src = bptr->invars[len];
 383                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 384                                         if (bptr->type == BBTYPE_EXH) {
 385 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 386                                                 if (!IS_INMEMORY(src->flags))
 387                                                         d= src->vv.regoff;
 388                                                 else
 389                                                         d=REG_ITMP1;
 390                                                 M_INTMOVE(REG_ITMP1, d);
 391                                                 emit_store(jd, NULL, src, d);
 392                                         }
 393                                 }
 394                         }
 395
 396                 } else {
 397 #endif
 398
 399                 while (len) {
 400                         len--;
 401                         var = VAR(bptr->invars[len]);
 402                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 403                                 if (bptr->type == BBTYPE_EXH) {
 404                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 405                                         M_INTMOVE(REG_ITMP1, d);
 406                                         emit_store(jd, NULL, var, d);
 407                                 }
 408                         }
 409                         else {
 410                                 assert((var->flags & INOUT));
 411                         }
 412                 }
 413 #if defined(ENABLE_LSRA)
 414                 }
 415 #endif
 416                 /* walk through all instructions */
 417
 418                 len = bptr->icount;
 419                 currentline = 0;
 420
 421                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 422                         if (iptr->line != currentline) {
 423                                 dseg_addlinenumber(cd, iptr->line);
 424                                 currentline = iptr->line;
 425                         }
 426
 427                         MCODECHECK(1024);                         /* 1KB should be enough */
 428
 429                 switch (iptr->opc) {
 430                 case ICMD_NOP:        /* ...  ==> ...                                 */
 431                 case ICMD_POP:        /* ..., value  ==> ...                          */
 432                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 433                 case ICMD_INLINE_START: /* internal ICMDs                         */
 434                 case ICMD_INLINE_END:
 435                         break;
 436
 437                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 438
 439                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 440                         M_TEST(s1);
 441                         M_BEQ(0);
 442                         codegen_add_nullpointerexception_ref(cd);
 443                         break;
 444
 445                 /* constant operations ************************************************/
 446
 447                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 448
 449                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 450                         ICONST(d, iptr->sx.val.i);
 451                         emit_store_dst(jd, iptr, d);
 452                         break;
 453
 454                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 455
 456                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 457                         LCONST(d, iptr->sx.val.l);
 458                         emit_store_dst(jd, iptr, d);
 459                         break;
 460
 461                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 462
 463                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 464                         disp = dseg_addfloat(cd, iptr->sx.val.f);
 465                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 466                         emit_store_dst(jd, iptr, d);
 467                         break;
 468
 469                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 470
 471                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 472                         disp = dseg_adddouble(cd, iptr->sx.val.d);
 473                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 474                         emit_store_dst(jd, iptr, d);
 475                         break;
 476
 477                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 478
 479                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 480
 481                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 482                                 cr = iptr->sx.val.c.ref;
 483
 484 /*                              PROFILE_CYCLE_STOP; */
 485
 486                                 codegen_add_patch_ref(cd, PATCHER_aconst, cr, 0);
 487
 488 /*                              PROFILE_CYCLE_START; */
 489
 490                                 M_MOV_IMM(NULL, d);
 491
 492                         } else {
 493                                 if (iptr->sx.val.anyptr == 0)
 494                                         M_CLR(d);
 495                                 else
 496                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 497                         }
 498                         emit_store_dst(jd, iptr, d);
 499                         break;
 500
 501
 502                 /* load/store/copy/move operations ************************************/
 503
 504                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 505                 case ICMD_ALOAD:      /* s1 = local variable                          */
 506                 case ICMD_LLOAD:
 507                 case ICMD_FLOAD:
 508                 case ICMD_DLOAD:
 509                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 510                 case ICMD_ASTORE:     /* dst = local variable                         */
 511                 case ICMD_LSTORE:
 512                 case ICMD_FSTORE:
 513                 case ICMD_DSTORE:
 514                 case ICMD_COPY:
 515                 case ICMD_MOVE:
 516
 517                         emit_copy(jd, iptr, VAROP(iptr->s1), VAROP(iptr->dst));
 518                         break;
 519
 520
 521                 /* integer operations *************************************************/
 522
 523                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 524
 525                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 526                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 527                         M_INTMOVE(s1, d);
 528                         M_INEG(d);
 529                         emit_store_dst(jd, iptr, d);
 530                         break;
 531
 532                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 533
 534                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 535                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 536                         M_INTMOVE(s1, d);
 537                         M_LNEG(d);
 538                         emit_store_dst(jd, iptr, d);
 539                         break;
 540
 541                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 542
 543                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 544                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 545                         M_ISEXT(s1, d);
 546                         emit_store_dst(jd, iptr, d);
 547                         break;
 548
 549                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 550
 551                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 552                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 553                         M_IMOV(s1, d);
 554                         emit_store_dst(jd, iptr, d);
 555                         break;
 556
 557                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 558
 559                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 560                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 561                         M_BSEXT(s1, d);
 562                         emit_store_dst(jd, iptr, d);
 563                         break;
 564
 565                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 566
 567                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 568                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 569                         M_CZEXT(s1, d);
 570                         emit_store_dst(jd, iptr, d);
 571                         break;
 572
 573                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 574
 575                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 576                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 577                         M_SSEXT(s1, d);
 578                         emit_store_dst(jd, iptr, d);
 579                         break;
 580
 581
 582                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 583
 584                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 585                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 586                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 587                         if (s2 == d)
 588                                 M_IADD(s1, d);
 589                         else {
 590                                 M_INTMOVE(s1, d);
 591                                 M_IADD(s2, d);
 592                         }
 593                         emit_store_dst(jd, iptr, d);
 594                         break;
 595
 596                 case ICMD_IINC:
 597                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 598                                       /* sx.val.i = constant                             */
 599
 600                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 601                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 602
 603                         /* Using inc and dec is not faster than add (tested with
 604                            sieve). */
 605
 606                         M_INTMOVE(s1, d);
 607                         M_IADD_IMM(iptr->sx.val.i, d);
 608                         emit_store_dst(jd, iptr, d);
 609                         break;
 610
 611                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 612
 613                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 614                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 615                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 616                         if (s2 == d)
 617                                 M_LADD(s1, d);
 618                         else {
 619                                 M_INTMOVE(s1, d);
 620                                 M_LADD(s2, d);
 621                         }
 622                         emit_store_dst(jd, iptr, d);
 623                         break;
 624
 625                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 626                                       /* sx.val.l = constant                             */
 627
 628                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 629                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 630                         M_INTMOVE(s1, d);
 631                         if (IS_IMM32(iptr->sx.val.l))
 632                                 M_LADD_IMM(iptr->sx.val.l, d);
 633                         else {
 634                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 635                                 M_LADD(REG_ITMP2, d);
 636                         }
 637                         emit_store_dst(jd, iptr, d);
 638                         break;
 639
 640                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 641
 642                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 643                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 644                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 645                         if (s2 == d) {
 646                                 M_INTMOVE(s1, REG_ITMP1);
 647                                 M_ISUB(s2, REG_ITMP1);
 648                                 M_INTMOVE(REG_ITMP1, d);
 649                         } else {
 650                                 M_INTMOVE(s1, d);
 651                                 M_ISUB(s2, d);
 652                         }
 653                         emit_store_dst(jd, iptr, d);
 654                         break;
 655
 656                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 657                                       /* sx.val.i = constant                             */
 658
 659                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 660                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 661                         M_INTMOVE(s1, d);
 662                         M_ISUB_IMM(iptr->sx.val.i, d);
 663                         emit_store_dst(jd, iptr, d);
 664                         break;
 665
 666                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 667
 668                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 669                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 670                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 671                         if (s2 == d) {
 672                                 M_INTMOVE(s1, REG_ITMP1);
 673                                 M_LSUB(s2, REG_ITMP1);
 674                                 M_INTMOVE(REG_ITMP1, d);
 675                         } else {
 676                                 M_INTMOVE(s1, d);
 677                                 M_LSUB(s2, d);
 678                         }
 679                         emit_store_dst(jd, iptr, d);
 680                         break;
 681
 682                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 683                                       /* sx.val.l = constant                             */
 684
 685                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 686                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 687                         M_INTMOVE(s1, d);
 688                         if (IS_IMM32(iptr->sx.val.l))
 689                                 M_LSUB_IMM(iptr->sx.val.l, d);
 690                         else {
 691                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 692                                 M_LSUB(REG_ITMP2, d);
 693                         }
 694                         emit_store_dst(jd, iptr, d);
 695                         break;
 696
 697                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 698
 699                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 700                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 701                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 702                         if (s2 == d)
 703                                 M_IMUL(s1, d);
 704                         else {
 705                                 M_INTMOVE(s1, d);
 706                                 M_IMUL(s2, d);
 707                         }
 708                         emit_store_dst(jd, iptr, d);
 709                         break;
 710
 711                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 712                                       /* sx.val.i = constant                             */
 713
 714                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 715                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 716                         if (iptr->sx.val.i == 2) {
 717                                 M_INTMOVE(s1, d);
 718                                 M_ISLL_IMM(1, d);
 719                         } else
 720                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 721                         emit_store_dst(jd, iptr, d);
 722                         break;
 723
 724                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 725
 726                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 727                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 728                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 729                         if (s2 == d)
 730                                 M_LMUL(s1, d);
 731                         else {
 732                                 M_INTMOVE(s1, d);
 733                                 M_LMUL(s2, d);
 734                         }
 735                         emit_store_dst(jd, iptr, d);
 736                         break;
 737
 738                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 739                                       /* sx.val.l = constant                             */
 740
 741                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 742                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 743                         if (IS_IMM32(iptr->sx.val.l))
 744                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 745                         else {
 746                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 747                                 M_INTMOVE(s1, d);
 748                                 M_LMUL(REG_ITMP2, d);
 749                         }
 750                         emit_store_dst(jd, iptr, d);
 751                         break;
 752
 753                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 754
 755                         var1 = VAROP(iptr->s1);
 756                         var2 = VAROP(iptr->sx.s23.s2);
 757                         dst  = VAROP(iptr->dst);
 758
 759                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 760                         if (IS_INMEMORY(var1->flags))
 761                                 M_ILD(RAX, REG_SP, var1->vv.regoff * 8);
 762                         else
 763                                 M_INTMOVE(var1->vv.regoff, RAX);
 764
 765                         if (IS_INMEMORY(var2->flags))
 766                                 M_ILD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 767                         else
 768                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 769
 770                         if (checknull) {
 771                                 M_ITEST(REG_ITMP3);
 772                                 M_BEQ(0);
 773                                 codegen_add_arithmeticexception_ref(cd);
 774                         }
 775
 776                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
 777                         emit_jcc(cd, CC_NE, 4 + 6);
 778                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
 779                         emit_jcc(cd, CC_E, 3 + 1 + 3);                /* 6 bytes */
 780
 781                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 782                         emit_cltd(cd);
 783                         emit_idivl_reg(cd, REG_ITMP3);
 784
 785                         if (IS_INMEMORY(dst->flags)) {
 786                                 emit_mov_reg_membase(cd, RAX, REG_SP, dst->vv.regoff * 8);
 787                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
 788
 789                         } else {
 790                                 M_INTMOVE(RAX, dst->vv.regoff);
 791
 792                                 if (dst->vv.regoff != RDX) {
 793                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
 794                                 }
 795                         }
 796                         break;
 797
 798                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 799
 800                         var1 = VAROP(iptr->s1);
 801                         var2 = VAROP(iptr->sx.s23.s2);
 802                         dst  = VAROP(iptr->dst);
 803
 804                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 805                         if (IS_INMEMORY(var1->flags))
 806                                 M_ILD(RAX, REG_SP, var1->vv.regoff * 8);
 807                         else
 808                                 M_INTMOVE(var1->vv.regoff, RAX);
 809
 810                         if (IS_INMEMORY(var2->flags))
 811                                 M_ILD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 812                         else
 813                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 814
 815                         if (checknull) {
 816                                 M_ITEST(REG_ITMP3);
 817                                 M_BEQ(0);
 818                                 codegen_add_arithmeticexception_ref(cd);
 819                         }
 820
 821                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 822
 823                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
 824                         emit_jcc(cd, CC_NE, 2 + 4 + 6);
 825
 826
 827                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
 828                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
 829                         emit_jcc(cd, CC_E, 1 + 3);                    /* 6 bytes */
 830
 831                         emit_cltd(cd);
 832                         emit_idivl_reg(cd, REG_ITMP3);
 833
 834                         if (IS_INMEMORY(dst->flags)) {
 835                                 emit_mov_reg_membase(cd, RDX, REG_SP, dst->vv.regoff * 8);
 836                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
 837
 838                         } else {
 839                                 M_INTMOVE(RDX, dst->vv.regoff);
 840
 841                                 if (dst->vv.regoff != RDX) {
 842                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
 843                                 }
 844                         }
 845                         break;
 846
 847                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 848                                       /* sx.val.i = constant                             */
 849
 850                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 851                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 852                         M_INTMOVE(s1, REG_ITMP1);
 853                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 854                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 855                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 856                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 857                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 858                         emit_store_dst(jd, iptr, d);
 859                         break;
 860
 861                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 862                                       /* sx.val.i = constant                             */
 863
 864                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 865                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 866                         M_INTMOVE(s1, REG_ITMP1);
 867                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 868                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 869                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 870                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 871                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 872                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 873                         emit_store_dst(jd, iptr, d);
 874                         break;
 875
 876
 877                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 878
 879                         var1 = VAROP(iptr->s1);
 880                         var2 = VAROP(iptr->sx.s23.s2);
 881                         dst  = VAROP(iptr->dst);
 882
 883                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 884
 885                 if (IS_INMEMORY(var1->flags))
 886                                 M_LLD(RAX, REG_SP, var1->vv.regoff * 8);
 887                         else
 888                                 M_INTMOVE(var1->vv.regoff, RAX);
 889
 890                         if (IS_INMEMORY(var2->flags))
 891                                 M_LLD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 892                         else
 893                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 894
 895                         if (checknull) {
 896                                 M_TEST(REG_ITMP3);
 897                                 M_BEQ(0);
 898                                 codegen_add_arithmeticexception_ref(cd);
 899                         }
 900
 901                         /* check as described in jvm spec */
 902                         disp = dseg_adds8(cd, 0x8000000000000000LL);
 903                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 904                         M_BNE(4 + 6);
 905                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
 906                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
 907
 908                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 909                         emit_cqto(cd);
 910                         emit_idiv_reg(cd, REG_ITMP3);
 911
 912                         if (IS_INMEMORY(dst->flags)) {
 913                                 M_LST(RAX, REG_SP, dst->vv.regoff * 8);
 914                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
 915
 916                         } else {
 917                                 M_INTMOVE(RAX, dst->vv.regoff);
 918
 919                                 if (dst->vv.regoff != RDX) {
 920                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
 921                                 }
 922                         }
 923                         break;
 924
 925                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 926
 927                         var1 = VAROP(iptr->s1);
 928                         var2 = VAROP(iptr->sx.s23.s2);
 929                         dst  = VAROP(iptr->dst);
 930
 931                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 932
 933                         if (IS_INMEMORY(var1->flags))
 934                                 M_LLD(REG_ITMP1, REG_SP, var1->vv.regoff * 8);
 935                         else
 936                                 M_INTMOVE(var1->vv.regoff, REG_ITMP1);
 937
 938                         if (IS_INMEMORY(var2->flags))
 939                                 M_LLD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 940                         else
 941                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 942                         /*
 943                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 944                         M_INTMOVE(s1, REG_ITMP1);
 945                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 946                         M_INTMOVE(s2, REG_ITMP3);
 947                         */
 948                         if (checknull) {
 949                                 M_ITEST(REG_ITMP3);
 950                                 M_BEQ(0);
 951                                 codegen_add_arithmeticexception_ref(cd);
 952                         }
 953
 954                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 955
 956                         /* check as described in jvm spec */
 957                         disp = dseg_adds8(cd, 0x8000000000000000LL);
 958                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 959                         M_BNE(3 + 4 + 6);
 960
 961 #if 0
 962                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
 963 #endif
 964                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 965                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
 966                         M_BEQ(2 + 3);                                          /* 6 bytes */
 967
 968                         emit_cqto(cd);
 969                         emit_idiv_reg(cd, REG_ITMP3);
 970
 971                         if (IS_INMEMORY(dst->flags)) {
 972                                 M_LST(RDX, REG_SP, dst->vv.regoff * 8);
 973                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
 974
 975                         } else {
 976                                 M_INTMOVE(RDX, dst->vv.regoff);
 977
 978                                 if (dst->vv.regoff != RDX) {
 979                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
 980                                 }
 981                         }
 982                         break;
 983
 984                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 985                                       /* sx.val.i = constant                             */
 986
 987                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 988                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 989                         M_INTMOVE(s1, REG_ITMP1);
 990                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 991                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 992                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 993                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 994                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 995                         emit_store_dst(jd, iptr, d);
 996                         break;
 997
 998                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 999                                       /* sx.val.l = constant                             */
1000
1001                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1002                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1003                         M_INTMOVE(s1, REG_ITMP1);
1004                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1005                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
1006                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
1007                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
1008                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
1009                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1010                         emit_store_dst(jd, iptr, d);
1011                         break;
1012
1013                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1014
1015                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1016                         emit_ishift(jd, SHIFT_SHL, iptr);
1017                         break;
1018
1019                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1020                                       /* sx.val.i = constant                             */
1021
1022                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1023                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1024                         M_INTMOVE(s1, d);
1025                         M_ISLL_IMM(iptr->sx.val.i, d);
1026                         emit_store_dst(jd, iptr, d);
1027                         break;
1028
1029                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1030
1031                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1032                         emit_ishift(jd, SHIFT_SAR, iptr);
1033                         break;
1034
1035                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1036                                       /* sx.val.i = constant                             */
1037
1038                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1039                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1040                         M_INTMOVE(s1, d);
1041                         M_ISRA_IMM(iptr->sx.val.i, d);
1042                         emit_store_dst(jd, iptr, d);
1043                         break;
1044
1045                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1046
1047                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1048                         emit_ishift(jd, SHIFT_SHR, iptr);
1049                         break;
1050
1051                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1052                                       /* sx.val.i = constant                             */
1053
1054                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1055                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1056                         M_INTMOVE(s1, d);
1057                         M_ISRL_IMM(iptr->sx.val.i, d);
1058                         emit_store_dst(jd, iptr, d);
1059                         break;
1060
1061                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1062
1063                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1064                         emit_lshift(jd, SHIFT_SHL, iptr);
1065                         break;
1066
1067         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1068                                           /* sx.val.i = constant                             */
1069
1070                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1071                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1072                         M_INTMOVE(s1, d);
1073                         M_LSLL_IMM(iptr->sx.val.i, d);
1074                         emit_store_dst(jd, iptr, d);
1075                         break;
1076
1077                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1078
1079                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1080                         emit_lshift(jd, SHIFT_SAR, iptr);
1081                         break;
1082
1083                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1084                                       /* sx.val.i = constant                             */
1085
1086                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1087                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1088                         M_INTMOVE(s1, d);
1089                         M_LSRA_IMM(iptr->sx.val.i, d);
1090                         emit_store_dst(jd, iptr, d);
1091                         break;
1092
1093                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1094
1095                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1096                         emit_lshift(jd, SHIFT_SHR, iptr);
1097                         break;
1098
1099                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1100                                       /* sx.val.l = constant                             */
1101
1102                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1103                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1104                         M_INTMOVE(s1, d);
1105                         M_LSRL_IMM(iptr->sx.val.i, d);
1106                         emit_store_dst(jd, iptr, d);
1107                         break;
1108
1109                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1110
1111                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1112                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1113                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1114                         if (s2 == d)
1115                                 M_IAND(s1, d);
1116                         else {
1117                                 M_INTMOVE(s1, d);
1118                                 M_IAND(s2, d);
1119                         }
1120                         emit_store_dst(jd, iptr, d);
1121                         break;
1122
1123                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1124                                       /* sx.val.i = constant                             */
1125
1126                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1127                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1128                         M_INTMOVE(s1, d);
1129                         M_IAND_IMM(iptr->sx.val.i, d);
1130                         emit_store_dst(jd, iptr, d);
1131                         break;
1132
1133                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1134
1135                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1136                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1137                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1138                         if (s2 == d)
1139                                 M_LAND(s1, d);
1140                         else {
1141                                 M_INTMOVE(s1, d);
1142                                 M_LAND(s2, d);
1143                         }
1144                         emit_store_dst(jd, iptr, d);
1145                         break;
1146
1147                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1148                                       /* sx.val.l = constant                             */
1149
1150                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1151                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1152                         M_INTMOVE(s1, d);
1153                         if (IS_IMM32(iptr->sx.val.l))
1154                                 M_LAND_IMM(iptr->sx.val.l, d);
1155                         else {
1156                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1157                                 M_LAND(REG_ITMP2, d);
1158                         }
1159                         emit_store_dst(jd, iptr, d);
1160                         break;
1161
1162                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1163
1164                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1165                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1166                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1167                         if (s2 == d)
1168                                 M_IOR(s1, d);
1169                         else {
1170                                 M_INTMOVE(s1, d);
1171                                 M_IOR(s2, d);
1172                         }
1173                         emit_store_dst(jd, iptr, d);
1174                         break;
1175
1176                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1177                                       /* sx.val.i = constant                             */
1178
1179                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1180                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1181                         M_INTMOVE(s1, d);
1182                         M_IOR_IMM(iptr->sx.val.i, d);
1183                         emit_store_dst(jd, iptr, d);
1184                         break;
1185
1186                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1187
1188                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1189                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1190                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1191                         if (s2 == d)
1192                                 M_LOR(s1, d);
1193                         else {
1194                                 M_INTMOVE(s1, d);
1195                                 M_LOR(s2, d);
1196                         }
1197                         emit_store_dst(jd, iptr, d);
1198                         break;
1199
1200                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1201                                       /* sx.val.l = constant                             */
1202
1203                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1204                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1205                         M_INTMOVE(s1, d);
1206                         if (IS_IMM32(iptr->sx.val.l))
1207                                 M_LOR_IMM(iptr->sx.val.l, d);
1208                         else {
1209                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1210                                 M_LOR(REG_ITMP2, d);
1211                         }
1212                         emit_store_dst(jd, iptr, d);
1213                         break;
1214
1215                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1216
1217                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1218                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1219                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1220                         if (s2 == d)
1221                                 M_IXOR(s1, d);
1222                         else {
1223                                 M_INTMOVE(s1, d);
1224                                 M_IXOR(s2, d);
1225                         }
1226                         emit_store_dst(jd, iptr, d);
1227                         break;
1228
1229                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1230                                       /* sx.val.i = constant                             */
1231
1232                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1233                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1234                         M_INTMOVE(s1, d);
1235                         M_IXOR_IMM(iptr->sx.val.i, d);
1236                         emit_store_dst(jd, iptr, d);
1237                         break;
1238
1239                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1240
1241                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1242                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1243                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1244                         if (s2 == d)
1245                                 M_LXOR(s1, d);
1246                         else {
1247                                 M_INTMOVE(s1, d);
1248                                 M_LXOR(s2, d);
1249                         }
1250                         emit_store_dst(jd, iptr, d);
1251                         break;
1252
1253                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1254                                       /* sx.val.l = constant                             */
1255
1256                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1257                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1258                         M_INTMOVE(s1, d);
1259                         if (IS_IMM32(iptr->sx.val.l))
1260                                 M_LXOR_IMM(iptr->sx.val.l, d);
1261                         else {
1262                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1263                                 M_LXOR(REG_ITMP2, d);
1264                         }
1265                         emit_store_dst(jd, iptr, d);
1266                         break;
1267
1268
1269                 /* floating operations ************************************************/
1270
1271                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1272
1273                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1274                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1275                         disp = dseg_adds4(cd, 0x80000000);
1276                         M_FLTMOVE(s1, d);
1277                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1278                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1279                         emit_store_dst(jd, iptr, d);
1280                         break;
1281
1282                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1283
1284                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1285                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1286                         disp = dseg_adds8(cd, 0x8000000000000000);
1287                         M_FLTMOVE(s1, d);
1288                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1289                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1290                         emit_store_dst(jd, iptr, d);
1291                         break;
1292
1293                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1294
1295                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1296                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1297                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1298                         if (s2 == d)
1299                                 M_FADD(s1, d);
1300                         else {
1301                                 M_FLTMOVE(s1, d);
1302                                 M_FADD(s2, d);
1303                         }
1304                         emit_store_dst(jd, iptr, d);
1305                         break;
1306
1307                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1308
1309                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1310                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1311                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1312                         if (s2 == d)
1313                                 M_DADD(s1, d);
1314                         else {
1315                                 M_FLTMOVE(s1, d);
1316                                 M_DADD(s2, d);
1317                         }
1318                         emit_store_dst(jd, iptr, d);
1319                         break;
1320
1321                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1322
1323                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1324                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1325                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1326                         if (s2 == d) {
1327                                 M_FLTMOVE(s2, REG_FTMP2);
1328                                 s2 = REG_FTMP2;
1329                         }
1330                         M_FLTMOVE(s1, d);
1331                         M_FSUB(s2, d);
1332                         emit_store_dst(jd, iptr, d);
1333                         break;
1334
1335                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1336
1337                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1338                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1339                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1340                         if (s2 == d) {
1341                                 M_FLTMOVE(s2, REG_FTMP2);
1342                                 s2 = REG_FTMP2;
1343                         }
1344                         M_FLTMOVE(s1, d);
1345                         M_DSUB(s2, d);
1346                         emit_store_dst(jd, iptr, d);
1347                         break;
1348
1349                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1350
1351                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1352                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1353                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1354                         if (s2 == d)
1355                                 M_FMUL(s1, d);
1356                         else {
1357                                 M_FLTMOVE(s1, d);
1358                                 M_FMUL(s2, d);
1359                         }
1360                         emit_store_dst(jd, iptr, d);
1361                         break;
1362
1363                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1364
1365                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1366                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1367                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1368                         if (s2 == d)
1369                                 M_DMUL(s1, d);
1370                         else {
1371                                 M_FLTMOVE(s1, d);
1372                                 M_DMUL(s2, d);
1373                         }
1374                         emit_store_dst(jd, iptr, d);
1375                         break;
1376
1377                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1378
1379                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1380                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1381                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1382                         if (s2 == d) {
1383                                 M_FLTMOVE(s2, REG_FTMP2);
1384                                 s2 = REG_FTMP2;
1385                         }
1386                         M_FLTMOVE(s1, d);
1387                         M_FDIV(s2, d);
1388                         emit_store_dst(jd, iptr, d);
1389                         break;
1390
1391                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1392
1393                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1394                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1395                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1396                         if (s2 == d) {
1397                                 M_FLTMOVE(s2, REG_FTMP2);
1398                                 s2 = REG_FTMP2;
1399                         }
1400                         M_FLTMOVE(s1, d);
1401                         M_DDIV(s2, d);
1402                         emit_store_dst(jd, iptr, d);
1403                         break;
1404
1405                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1406
1407                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1408                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1409                         M_CVTIF(s1, d);
1410                         emit_store_dst(jd, iptr, d);
1411                         break;
1412
1413                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1414
1415                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1416                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1417                         M_CVTID(s1, d);
1418                         emit_store_dst(jd, iptr, d);
1419                         break;
1420
1421                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1422
1423                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1424                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1425                         M_CVTLF(s1, d);
1426                         emit_store_dst(jd, iptr, d);
1427                         break;
1428
1429                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1430
1431                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1432                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1433                         M_CVTLD(s1, d);
1434                         emit_store_dst(jd, iptr, d);
1435                         break;
1436
1437                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1438
1439                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1440                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1441                         M_CVTFI(s1, d);
1442                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1443                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1444                                 ((REG_RESULT == d) ? 0 : 3);
1445                         M_BNE(disp);
1446                         M_FLTMOVE(s1, REG_FTMP1);
1447                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1448                         M_CALL(REG_ITMP2);
1449                         M_INTMOVE(REG_RESULT, d);
1450                         emit_store_dst(jd, iptr, d);
1451                         break;
1452
1453                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1454
1455                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1456                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1457                         M_CVTDI(s1, d);
1458                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1459                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1460                                 ((REG_RESULT == d) ? 0 : 3);
1461                         M_BNE(disp);
1462                         M_FLTMOVE(s1, REG_FTMP1);
1463                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1464                         M_CALL(REG_ITMP2);
1465                         M_INTMOVE(REG_RESULT, d);
1466                         emit_store_dst(jd, iptr, d);
1467                         break;
1468
1469                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1470
1471                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1472                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1473                         M_CVTFL(s1, d);
1474                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1475                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1476                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1477                                 ((REG_RESULT == d) ? 0 : 3);
1478                         M_BNE(disp);
1479                         M_FLTMOVE(s1, REG_FTMP1);
1480                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1481                         M_CALL(REG_ITMP2);
1482                         M_INTMOVE(REG_RESULT, d);
1483                         emit_store_dst(jd, iptr, d);
1484                         break;
1485
1486                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1487
1488                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1489                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1490                         M_CVTDL(s1, d);
1491                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1492                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1493                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1494                                 ((REG_RESULT == d) ? 0 : 3);
1495                         M_BNE(disp);
1496                         M_FLTMOVE(s1, REG_FTMP1);
1497                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1498                         M_CALL(REG_ITMP2);
1499                         M_INTMOVE(REG_RESULT, d);
1500                         emit_store_dst(jd, iptr, d);
1501                         break;
1502
1503                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1504
1505                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1506                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1507                         M_CVTFD(s1, d);
1508                         emit_store_dst(jd, iptr, d);
1509                         break;
1510
1511                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1512
1513                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1514                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1515                         M_CVTDF(s1, d);
1516                         emit_store_dst(jd, iptr, d);
1517                         break;
1518
1519                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1520                                           /* == => 0, < => 1, > => -1 */
1521
1522                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1523                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1524                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1525                         M_CLR(d);
1526                         M_MOV_IMM(1, REG_ITMP1);
1527                         M_MOV_IMM(-1, REG_ITMP2);
1528                         emit_ucomiss_reg_reg(cd, s1, s2);
1529                         M_CMOVB(REG_ITMP1, d);
1530                         M_CMOVA(REG_ITMP2, d);
1531                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1532                         emit_store_dst(jd, iptr, d);
1533                         break;
1534
1535                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1536                                           /* == => 0, < => 1, > => -1 */
1537
1538                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1539                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1540                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1541                         M_CLR(d);
1542                         M_MOV_IMM(1, REG_ITMP1);
1543                         M_MOV_IMM(-1, REG_ITMP2);
1544                         emit_ucomiss_reg_reg(cd, s1, s2);
1545                         M_CMOVB(REG_ITMP1, d);
1546                         M_CMOVA(REG_ITMP2, d);
1547                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1548                         emit_store_dst(jd, iptr, d);
1549                         break;
1550
1551                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1552                                           /* == => 0, < => 1, > => -1 */
1553
1554                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1555                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1556                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1557                         M_CLR(d);
1558                         M_MOV_IMM(1, REG_ITMP1);
1559                         M_MOV_IMM(-1, REG_ITMP2);
1560                         emit_ucomisd_reg_reg(cd, s1, s2);
1561                         M_CMOVB(REG_ITMP1, d);
1562                         M_CMOVA(REG_ITMP2, d);
1563                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1564                         emit_store_dst(jd, iptr, d);
1565                         break;
1566
1567                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1568                                           /* == => 0, < => 1, > => -1 */
1569
1570                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1571                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1572                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1573                         M_CLR(d);
1574                         M_MOV_IMM(1, REG_ITMP1);
1575                         M_MOV_IMM(-1, REG_ITMP2);
1576                         emit_ucomisd_reg_reg(cd, s1, s2);
1577                         M_CMOVB(REG_ITMP1, d);
1578                         M_CMOVA(REG_ITMP2, d);
1579                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1580                         emit_store_dst(jd, iptr, d);
1581                         break;
1582
1583
1584                 /* memory operations **************************************************/
1585
1586                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1587
1588                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1589                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1590                         gen_nullptr_check(s1);
1591                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1592                         emit_store_dst(jd, iptr, d);
1593                         break;
1594
1595                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1596
1597                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1598                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1599                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1600                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1601                                 gen_nullptr_check(s1);
1602                                 gen_bound_check;
1603                         }
1604                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1605                         emit_store_dst(jd, iptr, d);
1606                         break;
1607
1608                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1609
1610                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1611                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1612                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1613                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1614                                 gen_nullptr_check(s1);
1615                                 gen_bound_check;
1616                         }
1617                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1618                         emit_store_dst(jd, iptr, d);
1619                         break;
1620
1621                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1622
1623                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1624                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1625                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1626                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1627                                 gen_nullptr_check(s1);
1628                                 gen_bound_check;
1629                         }
1630                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1631                         emit_store_dst(jd, iptr, d);
1632                         break;
1633
1634                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1635
1636                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1637                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1638                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1639                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1640                                 gen_nullptr_check(s1);
1641                                 gen_bound_check;
1642                         }
1643                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1644                         emit_store_dst(jd, iptr, d);
1645                         break;
1646
1647                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1648
1649                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1650                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1651                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1652                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1653                                 gen_nullptr_check(s1);
1654                                 gen_bound_check;
1655                         }
1656                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1657                         emit_store_dst(jd, iptr, d);
1658                         break;
1659
1660                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1661
1662                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1663                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1664                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1665                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1666                                 gen_nullptr_check(s1);
1667                                 gen_bound_check;
1668                         }
1669                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1670                         emit_store_dst(jd, iptr, d);
1671                         break;
1672
1673                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1674
1675                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1676                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1677                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1678                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1679                                 gen_nullptr_check(s1);
1680                                 gen_bound_check;
1681                         }
1682                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1683                         emit_store_dst(jd, iptr, d);
1684                         break;
1685
1686                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1687
1688                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1689                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1690                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1691                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1692                                 gen_nullptr_check(s1);
1693                                 gen_bound_check;
1694                         }
1695                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1696                         emit_store_dst(jd, iptr, d);
1697                         break;
1698
1699
1700                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1701
1702                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1703                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1704                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1705                                 gen_nullptr_check(s1);
1706                                 gen_bound_check;
1707                         }
1708                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1709                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1710                         break;
1711
1712                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1713
1714                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1715                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1716                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1717                                 gen_nullptr_check(s1);
1718                                 gen_bound_check;
1719                         }
1720                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1721                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1722                         break;
1723
1724                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1725
1726                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1727                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1728                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1729                                 gen_nullptr_check(s1);
1730                                 gen_bound_check;
1731                         }
1732                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1733                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1734                         break;
1735
1736                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1737
1738                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1739                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1740                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1741                                 gen_nullptr_check(s1);
1742                                 gen_bound_check;
1743                         }
1744                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1745                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
1746                         break;
1747
1748                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1749
1750                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1751                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1752                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1753                                 gen_nullptr_check(s1);
1754                                 gen_bound_check;
1755                         }
1756                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1757                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
1758                         break;
1759
1760                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1761
1762                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1763                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1764                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1765                                 gen_nullptr_check(s1);
1766                                 gen_bound_check;
1767                         }
1768                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1769                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
1770                         break;
1771
1772                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1773
1774                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1775                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1776                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1777                                 gen_nullptr_check(s1);
1778                                 gen_bound_check;
1779                         }
1780                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1781                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
1782                         break;
1783
1784                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1785
1786                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1787                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1788                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1789                                 gen_nullptr_check(s1);
1790                                 gen_bound_check;
1791                         }
1792                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1793
1794                         M_MOV(s1, REG_A0);
1795                         M_MOV(s3, REG_A1);
1796                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
1797                         M_CALL(REG_ITMP1);
1798                         M_TEST(REG_RESULT);
1799                         M_BEQ(0);
1800                         codegen_add_arraystoreexception_ref(cd);
1801
1802                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1803                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1804                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1805                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1806                         break;
1807
1808
1809                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1810
1811                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1812                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1813                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1814                                 gen_nullptr_check(s1);
1815                                 gen_bound_check;
1816                         }
1817                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1818                         break;
1819
1820                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1821
1822                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1823                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1824                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1825                                 gen_nullptr_check(s1);
1826                                 gen_bound_check;
1827                         }
1828                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray, data[0]), s1, s2, 1);
1829                         break;
1830
1831                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1832
1833                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1834                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1835                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1836                                 gen_nullptr_check(s1);
1837                                 gen_bound_check;
1838                         }
1839                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1840                         break;
1841
1842                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1843
1844                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1845                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1846                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1847                                 gen_nullptr_check(s1);
1848                                 gen_bound_check;
1849                         }
1850                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray, data[0]), s1, s2, 2);
1851                         break;
1852
1853                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1854
1855                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1856                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1857                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1858                                 gen_nullptr_check(s1);
1859                                 gen_bound_check;
1860                         }
1861
1862                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1863                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1864                         } else {
1865                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1866                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
1867                         }
1868                         break;
1869
1870                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1871
1872                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1873                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1874                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1875                                 gen_nullptr_check(s1);
1876                                 gen_bound_check;
1877                         }
1878                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1879                         break;
1880
1881
1882                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1883
1884                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1885                                 uf        = iptr->sx.s23.s3.uf;
1886                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1887                                 disp      = dseg_addaddress(cd, NULL);
1888                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1889
1890                                 /* must be calculated before codegen_add_patch_ref */
1891
1892                                 if (opt_shownops)
1893                                         disp -= PATCHER_CALL_SIZE;
1894
1895 /*                              PROFILE_CYCLE_STOP; */
1896
1897                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1898
1899 /*                              PROFILE_CYCLE_START; */
1900                         }
1901                         else {
1902                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1903                                 fieldtype = fi->type;
1904                                 disp      = dseg_addaddress(cd, &(fi->value));
1905                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1906
1907                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1908                                         PROFILE_CYCLE_STOP;
1909
1910                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1911
1912                                         if (opt_shownops)
1913                                                 disp -= PATCHER_CALL_SIZE;
1914
1915                                         PROFILE_CYCLE_START;
1916                                 }
1917                         }
1918
1919                         /* This approach is much faster than moving the field
1920                            address inline into a register. */
1921
1922                         M_ALD(REG_ITMP1, RIP, disp);
1923
1924                         switch (fieldtype) {
1925                         case TYPE_INT:
1926                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1927                                 M_ILD(d, REG_ITMP1, 0);
1928                                 break;
1929                         case TYPE_LNG:
1930                         case TYPE_ADR:
1931                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1932                                 M_LLD(d, REG_ITMP1, 0);
1933                                 break;
1934                         case TYPE_FLT:
1935                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1936                                 M_FLD(d, REG_ITMP1, 0);
1937                                 break;
1938                         case TYPE_DBL:
1939                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1940                                 M_DLD(d, REG_ITMP1, 0);
1941                                 break;
1942                         }
1943                         emit_store_dst(jd, iptr, d);
1944                         break;
1945
1946                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1947
1948                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1949                                 uf        = iptr->sx.s23.s3.uf;
1950                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1951                                 disp      = dseg_addaddress(cd, NULL);
1952                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1953
1954                                 /* must be calculated before codegen_add_patch_ref */
1955
1956                                 if (opt_shownops)
1957                                         disp -= PATCHER_CALL_SIZE;
1958
1959 /*                              PROFILE_CYCLE_STOP; */
1960
1961                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1962
1963 /*                              PROFILE_CYCLE_START; */
1964                         }
1965                         else {
1966                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1967                                 fieldtype = fi->type;
1968                                 disp      = dseg_addaddress(cd, &(fi->value));
1969                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1970
1971                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1972                                         PROFILE_CYCLE_STOP;
1973
1974                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1975
1976                                         if (opt_shownops)
1977                                                 disp -= PATCHER_CALL_SIZE;
1978
1979                                         PROFILE_CYCLE_START;
1980                                 }
1981                         }
1982
1983                         /* This approach is much faster than moving the field
1984                            address inline into a register. */
1985
1986                         M_ALD(REG_ITMP1, RIP, disp);
1987
1988                         switch (fieldtype) {
1989                         case TYPE_INT:
1990                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1991                                 M_IST(s1, REG_ITMP1, 0);
1992                                 break;
1993                         case TYPE_LNG:
1994                         case TYPE_ADR:
1995                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1996                                 M_LST(s1, REG_ITMP1, 0);
1997                                 break;
1998                         case TYPE_FLT:
1999                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
2000                                 M_FST(s1, REG_ITMP1, 0);
2001                                 break;
2002                         case TYPE_DBL:
2003                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
2004                                 M_DST(s1, REG_ITMP1, 0);
2005                                 break;
2006                         }
2007                         break;
2008
2009                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2010                                           /* val = value (in current instruction)     */
2011                                           /* following NOP)                           */
2012
2013                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2014                                 uf        = iptr->sx.s23.s3.uf;
2015                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2016                                 disp      = dseg_addaddress(cd, NULL);
2017                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2018
2019                                 /* must be calculated before codegen_add_patch_ref */
2020
2021                                 if (opt_shownops)
2022                                         disp -= PATCHER_CALL_SIZE;
2023
2024
2025 /*                              PROFILE_CYCLE_STOP; */
2026
2027                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
2028
2029 /*                              PROFILE_CYCLE_START; */
2030                         }
2031                         else {
2032                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2033                                 fieldtype = fi->type;
2034                                 disp      = dseg_addaddress(cd, &(fi->value));
2035                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2036
2037                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2038                                         PROFILE_CYCLE_STOP;
2039
2040                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
2041
2042                                         if (opt_shownops)
2043                                                 disp -= PATCHER_CALL_SIZE;
2044
2045                                         PROFILE_CYCLE_START;
2046                                 }
2047                         }
2048
2049                         /* This approach is much faster than moving the field
2050                            address inline into a register. */
2051
2052                         M_ALD(REG_ITMP1, RIP, disp);
2053
2054                         switch (fieldtype) {
2055                         case TYPE_INT:
2056                         case TYPE_FLT:
2057                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2058                                 break;
2059                         case TYPE_LNG:
2060                         case TYPE_ADR:
2061                         case TYPE_DBL:
2062                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
2063                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2064                                 else {
2065                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2066                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
2067                                 }
2068                                 break;
2069                         }
2070                         break;
2071
2072                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2073
2074                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2075                         gen_nullptr_check(s1);
2076
2077                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2078                                 uf        = iptr->sx.s23.s3.uf;
2079                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2080                                 disp      = 0;
2081
2082 /*                              PROFILE_CYCLE_STOP; */
2083
2084                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
2085
2086 /*                              PROFILE_CYCLE_START; */
2087                         }
2088                         else {
2089                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2090                                 fieldtype = fi->type;
2091                                 disp      = fi->offset;
2092                         }
2093
2094                         switch (fieldtype) {
2095                         case TYPE_INT:
2096                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
2097                                 M_ILD32(d, s1, disp);
2098                                 break;
2099                         case TYPE_LNG:
2100                         case TYPE_ADR:
2101                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
2102                                 M_LLD32(d, s1, disp);
2103                                 break;
2104                         case TYPE_FLT:
2105                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2106                                 M_FLD32(d, s1, disp);
2107                                 break;
2108                         case TYPE_DBL:
2109                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2110                                 M_DLD32(d, s1, disp);
2111                                 break;
2112                         }
2113                         emit_store_dst(jd, iptr, d);
2114                         break;
2115
2116                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2117
2118                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2119                         gen_nullptr_check(s1);
2120
2121                         s2 = emit_load_s2(jd, iptr, REG_IFTMP);
2122
2123                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2124                                 uf        = iptr->sx.s23.s3.uf;
2125                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2126                                 disp      = 0;
2127
2128 /*                              PROFILE_CYCLE_STOP; */
2129
2130                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
2131
2132 /*                              PROFILE_CYCLE_START; */
2133                         }
2134                         else {
2135                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2136                                 fieldtype = fi->type;
2137                                 disp      = fi->offset;
2138                         }
2139
2140                         switch (fieldtype) {
2141                         case TYPE_INT:
2142                                 M_IST32(s2, s1, disp);
2143                                 break;
2144                         case TYPE_LNG:
2145                         case TYPE_ADR:
2146                                 M_LST32(s2, s1, disp);
2147                                 break;
2148                         case TYPE_FLT:
2149                                 M_FST32(s2, s1, disp);
2150                                 break;
2151                         case TYPE_DBL:
2152                                 M_DST32(s2, s1, disp);
2153                                 break;
2154                         }
2155                         break;
2156
2157                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2158                                           /* val = value (in current instruction)     */
2159                                           /* following NOP)                           */
2160
2161                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2162                         gen_nullptr_check(s1);
2163
2164                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2165                                 uf        = iptr->sx.s23.s3.uf;
2166                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2167                                 disp      = 0;
2168
2169 /*                              PROFILE_CYCLE_STOP; */
2170
2171                                 codegen_add_patch_ref(cd, PATCHER_putfieldconst, uf, 0);
2172
2173 /*                              PROFILE_CYCLE_START; */
2174                         }
2175                         else {
2176                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2177                                 fieldtype = fi->type;
2178                                 disp      = fi->offset;
2179                         }
2180
2181                         switch (fieldtype) {
2182                         case TYPE_INT:
2183                         case TYPE_FLT:
2184                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2185                                 break;
2186                         case TYPE_LNG:
2187                         case TYPE_ADR:
2188                         case TYPE_DBL:
2189                                 /* XXX why no check for IS_IMM32? */
2190                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2191                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2192                                 break;
2193                         }
2194                         break;
2195
2196
2197                 /* branch operations **************************************************/
2198
2199                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2200
2201                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2202                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2203
2204                         PROFILE_CYCLE_STOP;
2205
2206 #ifdef ENABLE_VERIFIER
2207                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2208                                 uc = iptr->sx.s23.s2.uc;
2209
2210                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2211                         }
2212 #endif /* ENABLE_VERIFIER */
2213
2214                         M_CALL_IMM(0);                            /* passing exception pc */
2215                         M_POP(REG_ITMP2_XPC);
2216
2217                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2218                         M_JMP(REG_ITMP3);
2219                         break;
2220
2221                 case ICMD_GOTO:         /* ... ==> ...                                */
2222                 case ICMD_RET:          /* ... ==> ...                                */
2223
2224                         M_JMP_IMM(0);
2225                         codegen_addreference(cd, iptr->dst.block);
2226                         break;
2227
2228                 case ICMD_JSR:          /* ... ==> ...                                */
2229
2230                         M_JMP_IMM(0);
2231                         codegen_addreference(cd, iptr->sx.s23.s3.jsrtarget.block);
2232                         break;
2233
2234                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2235
2236                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2237                         M_TEST(s1);
2238                         M_BEQ(0);
2239                         codegen_addreference(cd, iptr->dst.block);
2240                         break;
2241
2242                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2243
2244                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2245                         M_TEST(s1);
2246                         M_BNE(0);
2247                         codegen_addreference(cd, iptr->dst.block);
2248                         break;
2249
2250                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2251
2252                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2253                         M_ICMP_IMM(iptr->sx.val.i, s1);
2254                         M_BEQ(0);
2255                         codegen_addreference(cd, iptr->dst.block);
2256                         break;
2257
2258                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2259
2260                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2261                         M_ICMP_IMM(iptr->sx.val.i, s1);
2262                         M_BLT(0);
2263                         codegen_addreference(cd, iptr->dst.block);
2264                         break;
2265
2266                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2267
2268                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2269                         M_ICMP_IMM(iptr->sx.val.i, s1);
2270                         M_BLE(0);
2271                         codegen_addreference(cd, iptr->dst.block);
2272                         break;
2273
2274                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2275
2276                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2277                         M_ICMP_IMM(iptr->sx.val.i, s1);
2278                         M_BNE(0);
2279                         codegen_addreference(cd, iptr->dst.block);
2280                         break;
2281
2282                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2283
2284                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2285                         M_ICMP_IMM(iptr->sx.val.i, s1);
2286                         M_BGT(0);
2287                         codegen_addreference(cd, iptr->dst.block);
2288                         break;
2289
2290                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2291
2292                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2293                         M_ICMP_IMM(iptr->sx.val.i, s1);
2294                         M_BGE(0);
2295                         codegen_addreference(cd, iptr->dst.block);
2296                         break;
2297
2298                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2299
2300                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2301                         if (IS_IMM32(iptr->sx.val.l))
2302                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2303                         else {
2304                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2305                                 M_LCMP(REG_ITMP2, s1);
2306                         }
2307                         M_BEQ(0);
2308                         codegen_addreference(cd, iptr->dst.block);
2309                         break;
2310
2311                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2312
2313                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2314                         if (IS_IMM32(iptr->sx.val.l))
2315                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2316                         else {
2317                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2318                                 M_LCMP(REG_ITMP2, s1);
2319                         }
2320                         M_BLT(0);
2321                         codegen_addreference(cd, iptr->dst.block);
2322                         break;
2323
2324                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2325
2326                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2327                         if (IS_IMM32(iptr->sx.val.l))
2328                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2329                         else {
2330                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2331                                 M_LCMP(REG_ITMP2, s1);
2332                         }
2333                         M_BLE(0);
2334                         codegen_addreference(cd, iptr->dst.block);
2335                         break;
2336
2337                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2338
2339                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2340                         if (IS_IMM32(iptr->sx.val.l))
2341                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2342                         else {
2343                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2344                                 M_LCMP(REG_ITMP2, s1);
2345                         }
2346                         M_BNE(0);
2347                         codegen_addreference(cd, iptr->dst.block);
2348                         break;
2349
2350                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2351
2352                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2353                         if (IS_IMM32(iptr->sx.val.l))
2354                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2355                         else {
2356                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2357                                 M_LCMP(REG_ITMP2, s1);
2358                         }
2359                         M_BGT(0);
2360                         codegen_addreference(cd, iptr->dst.block);
2361                         break;
2362
2363                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2364
2365                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2366                         if (IS_IMM32(iptr->sx.val.l))
2367                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2368                         else {
2369                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2370                                 M_LCMP(REG_ITMP2, s1);
2371                         }
2372                         M_BGE(0);
2373                         codegen_addreference(cd, iptr->dst.block);
2374                         break;
2375
2376                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2377
2378                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2379                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2380                         M_ICMP(s2, s1);
2381                         M_BEQ(0);
2382                         codegen_addreference(cd, iptr->dst.block);
2383                         break;
2384
2385                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2386                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2387
2388                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2389                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2390                         M_LCMP(s2, s1);
2391                         M_BEQ(0);
2392                         codegen_addreference(cd, iptr->dst.block);
2393                         break;
2394
2395                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2396
2397                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2398                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2399                         M_ICMP(s2, s1);
2400                         M_BNE(0);
2401                         codegen_addreference(cd, iptr->dst.block);
2402                         break;
2403
2404                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2405                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2406
2407                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2408                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2409                         M_LCMP(s2, s1);
2410                         M_BNE(0);
2411                         codegen_addreference(cd, iptr->dst.block);
2412                         break;
2413
2414                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2415
2416                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2417                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2418                         M_ICMP(s2, s1);
2419                         M_BLT(0);
2420                         codegen_addreference(cd, iptr->dst.block);
2421                         break;
2422
2423                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2424
2425                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2426                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2427                         M_LCMP(s2, s1);
2428                         M_BLT(0);
2429                         codegen_addreference(cd, iptr->dst.block);
2430                         break;
2431
2432                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2433
2434                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2435                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2436                         M_ICMP(s2, s1);
2437                         M_BGT(0);
2438                         codegen_addreference(cd, iptr->dst.block);
2439                         break;
2440
2441                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2442
2443                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2444                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2445                         M_LCMP(s2, s1);
2446                         M_BGT(0);
2447                         codegen_addreference(cd, iptr->dst.block);
2448                         break;
2449
2450                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2451
2452                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2453                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2454                         M_ICMP(s2, s1);
2455                         M_BLE(0);
2456                         codegen_addreference(cd, iptr->dst.block);
2457                         break;
2458
2459                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2460
2461                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2462                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2463                         M_LCMP(s2, s1);
2464                         M_BLE(0);
2465                         codegen_addreference(cd, iptr->dst.block);
2466                         break;
2467
2468                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2469
2470                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2471                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2472                         M_ICMP(s2, s1);
2473                         M_BGE(0);
2474                         codegen_addreference(cd, iptr->dst.block);
2475                         break;
2476
2477                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2478
2479                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2480                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2481                         M_LCMP(s2, s1);
2482                         M_BGE(0);
2483                         codegen_addreference(cd, iptr->dst.block);
2484                         break;
2485
2486                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2487                 case ICMD_LRETURN:
2488
2489                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2490                         M_INTMOVE(s1, REG_RESULT);
2491                         goto nowperformreturn;
2492
2493                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2494
2495                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2496                         M_INTMOVE(s1, REG_RESULT);
2497
2498 #ifdef ENABLE_VERIFIER
2499                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2500                                 uc = iptr->sx.s23.s2.uc;
2501
2502                                 PROFILE_CYCLE_STOP;
2503
2504                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2505
2506                                 PROFILE_CYCLE_START;
2507                         }
2508 #endif /* ENABLE_VERIFIER */
2509                         goto nowperformreturn;
2510
2511                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2512                 case ICMD_DRETURN:
2513
2514                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2515                         M_FLTMOVE(s1, REG_FRESULT);
2516                         goto nowperformreturn;
2517
2518                 case ICMD_RETURN:      /* ...  ==> ...                                */
2519
2520 nowperformreturn:
2521                         {
2522                         s4 i, p;
2523
2524                         p = cd->stackframesize;
2525
2526 #if !defined(NDEBUG)
2527                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2528                                 emit_verbosecall_exit(jd);
2529 #endif /* !defined(NDEBUG) */
2530
2531 #if defined(ENABLE_THREADS)
2532                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2533                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2534
2535                                 /* we need to save the proper return value */
2536                                 switch (iptr->opc) {
2537                                 case ICMD_IRETURN:
2538                                 case ICMD_ARETURN:
2539                                 case ICMD_LRETURN:
2540                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2541                                         break;
2542                                 case ICMD_FRETURN:
2543                                 case ICMD_DRETURN:
2544                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2545                                         break;
2546                                 }
2547
2548                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2549                                 M_CALL(REG_ITMP1);
2550
2551                                 /* and now restore the proper return value */
2552                                 switch (iptr->opc) {
2553                                 case ICMD_IRETURN:
2554                                 case ICMD_ARETURN:
2555                                 case ICMD_LRETURN:
2556                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2557                                         break;
2558                                 case ICMD_FRETURN:
2559                                 case ICMD_DRETURN:
2560                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2561                                         break;
2562                                 }
2563                         }
2564 #endif
2565
2566                         /* restore saved registers */
2567
2568                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2569                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2570                         }
2571                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2572                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2573                         }
2574
2575                         /* deallocate stack */
2576
2577                         if (cd->stackframesize)
2578                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2579
2580                         /* generate method profiling code */
2581
2582                         PROFILE_CYCLE_STOP;
2583
2584                         M_RET;
2585                         }
2586                         break;
2587
2588
2589                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2590                         {
2591                                 s4 i, l;
2592                                 branch_target_t *table;
2593
2594                                 table = iptr->dst.table;
2595
2596                                 l = iptr->sx.s23.s2.tablelow;
2597                                 i = iptr->sx.s23.s3.tablehigh;
2598
2599                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2600                                 M_INTMOVE(s1, REG_ITMP1);
2601
2602                                 if (l != 0)
2603                                         M_ISUB_IMM(l, REG_ITMP1);
2604
2605                                 /* number of targets */
2606                                 i = i - l + 1;
2607
2608                 /* range check */
2609                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2610                                 M_BA(0);
2611
2612                                 codegen_addreference(cd, table[0].block); /* default target */
2613
2614                                 /* build jump table top down and use address of lowest entry */
2615
2616                                 table += i;
2617
2618                                 while (--i >= 0) {
2619                                         dseg_addtarget(cd, table->block);
2620                                         --table;
2621                                 }
2622
2623                                 /* length of dataseg after last dseg_addtarget is used
2624                                    by load */
2625
2626                                 M_MOV_IMM(0, REG_ITMP2);
2627                                 dseg_adddata(cd);
2628                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2629                                 M_JMP(REG_ITMP1);
2630                         }
2631                         break;
2632
2633
2634                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2635                         {
2636                                 s4 i;
2637                                 lookup_target_t *lookup;
2638
2639                                 lookup = iptr->dst.lookup;
2640
2641                                 i = iptr->sx.s23.s2.lookupcount;
2642
2643                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2644                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2645
2646                                 while (--i >= 0) {
2647                                         M_ICMP_IMM(lookup->value, s1);
2648                                         M_BEQ(0);
2649                                         codegen_addreference(cd, lookup->target.block);
2650                                         lookup++;
2651                                 }
2652
2653                                 M_JMP_IMM(0);
2654
2655                                 codegen_addreference(cd, iptr->sx.s23.s3.lookupdefault.block);
2656                         }
2657                         break;
2658
2659
2660                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2661
2662                         bte = iptr->sx.s23.s3.bte;
2663                         md  = bte->md;
2664                         goto gen_method;
2665
2666                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2667
2668                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2669                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2670                 case ICMD_INVOKEINTERFACE:
2671
2672                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2673                                 lm = NULL;
2674                                 um = iptr->sx.s23.s3.um;
2675                                 md = um->methodref->parseddesc.md;
2676                         }
2677                         else {
2678                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2679                                 um = NULL;
2680                                 md = lm->parseddesc;
2681                         }
2682
2683 gen_method:
2684                         s3 = md->paramcount;
2685
2686                         MCODECHECK((20 * s3) + 128);
2687
2688                         /* copy arguments to registers or stack location */
2689
2690                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2691                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2692
2693                                 /* Already Preallocated (ARGVAR) ? */
2694                                 if (var->flags & PREALLOC)
2695                                         continue;
2696
2697                                 if (IS_INT_LNG_TYPE(var->type)) {
2698                                         if (!md->params[s3].inmemory) {
2699                                                 s1 = rd->argintregs[md->params[s3].regoff];
2700                                                 d = emit_load(jd, iptr, var, s1);
2701                                                 M_INTMOVE(d, s1);
2702                                         }
2703                                         else {
2704                                                 d = emit_load(jd, iptr, var, REG_ITMP1);
2705                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
2706                                         }
2707                                 }
2708                                 else {
2709                                         if (!md->params[s3].inmemory) {
2710                                                 s1 = rd->argfltregs[md->params[s3].regoff];
2711                                                 d = emit_load(jd, iptr, var, s1);
2712                                                 M_FLTMOVE(d, s1);
2713                                         }
2714                                         else {
2715                                                 d = emit_load(jd, iptr, var, REG_FTMP1);
2716
2717                                                 if (IS_2_WORD_TYPE(var->type))
2718                                                         M_DST(d, REG_SP, md->params[s3].regoff * 8);
2719                                                 else
2720                                                         M_FST(d, REG_SP, md->params[s3].regoff * 8);
2721                                         }
2722                                 }
2723                         }
2724
2725                         /* generate method profiling code */
2726
2727                         PROFILE_CYCLE_STOP;
2728
2729                         switch (iptr->opc) {
2730                         case ICMD_BUILTIN:
2731                                 M_MOV_IMM(bte->fp, REG_ITMP1);
2732                                 M_CALL(REG_ITMP1);
2733
2734                                 if (INSTRUCTION_MUST_CHECK(iptr)) {
2735                                         M_TEST(REG_RESULT);
2736                                         M_BEQ(0);
2737                                         codegen_add_fillinstacktrace_ref(cd);
2738                                 }
2739                                 break;
2740
2741                         case ICMD_INVOKESPECIAL:
2742                                 M_TEST(REG_A0);
2743                                 M_BEQ(0);
2744                                 codegen_add_nullpointerexception_ref(cd);
2745
2746                                 /* fall through */
2747
2748                         case ICMD_INVOKESTATIC:
2749                                 if (lm == NULL) {
2750                                         disp = dseg_addaddress(cd, NULL);
2751                                         disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2752
2753                                         /* must be calculated before codegen_add_patch_ref */
2754
2755                                         if (opt_shownops)
2756                                                 disp -= PATCHER_CALL_SIZE;
2757
2758                                         codegen_add_patch_ref(cd, PATCHER_invokestatic_special,
2759                                                                                   um, disp);
2760
2761 /*                                      a = 0; */
2762                                 }
2763                                 else {
2764                                         disp = dseg_addaddress(cd, lm->stubroutine);
2765                                         disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2766
2767 /*                                      a = (ptrint) lm->stubroutine; */
2768                                 }
2769
2770 /*                              M_MOV_IMM(a, REG_ITMP2); */
2771                                 M_ALD(REG_ITMP2, RIP, disp);
2772                                 M_CALL(REG_ITMP2);
2773                                 break;
2774
2775                         case ICMD_INVOKEVIRTUAL:
2776                                 gen_nullptr_check(REG_A0);
2777
2778                                 if (lm == NULL) {
2779                                         codegen_add_patch_ref(cd, PATCHER_invokevirtual, um, 0);
2780
2781                                         s1 = 0;
2782                                 }
2783                                 else
2784                                         s1 = OFFSET(vftbl_t, table[0]) +
2785                                                 sizeof(methodptr) * lm->vftblindex;
2786
2787                                 M_ALD(REG_METHODPTR, REG_A0,
2788                                           OFFSET(java_objectheader, vftbl));
2789                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2790                                 M_CALL(REG_ITMP3);
2791                                 break;
2792
2793                         case ICMD_INVOKEINTERFACE:
2794                                 gen_nullptr_check(REG_A0);
2795
2796                                 if (lm == NULL) {
2797                                         codegen_add_patch_ref(cd, PATCHER_invokeinterface, um, 0);
2798
2799                                         s1 = 0;
2800                                         s2 = 0;
2801                                 }
2802                                 else {
2803                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2804                                                 sizeof(methodptr) * lm->class->index;
2805
2806                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2807                                 }
2808
2809                                 M_ALD(REG_METHODPTR, REG_A0,
2810                                           OFFSET(java_objectheader, vftbl));
2811                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2812                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2813                                 M_CALL(REG_ITMP3);
2814                                 break;
2815                         }
2816
2817                         /* generate method profiling code */
2818
2819                         PROFILE_CYCLE_START;
2820
2821                         /* store return value */
2822
2823                         d = md->returntype.type;
2824
2825                         if (d != TYPE_VOID) {
2826                                 if (IS_INT_LNG_TYPE(d)) {
2827                                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2828                                         M_INTMOVE(REG_RESULT, s1);
2829                                 }
2830                                 else {
2831                                         s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2832                                         M_FLTMOVE(REG_FRESULT, s1);
2833                                 }
2834                                 emit_store_dst(jd, iptr, s1);
2835                         }
2836                         break;
2837
2838
2839                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2840
2841                                       /* val.a: (classinfo *) superclass              */
2842
2843                         /*  superclass is an interface:
2844                          *
2845                          *  OK if ((sub == NULL) ||
2846                          *         (sub->vftbl->interfacetablelength > super->index) &&
2847                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
2848                          *
2849                          *  superclass is a class:
2850                          *
2851                          *  OK if ((sub == NULL) || (0
2852                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
2853                          *         super->vftbl->diffval));
2854                          */
2855
2856                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2857                                 /* object type cast-check */
2858
2859                                 classinfo *super;
2860                                 vftbl_t   *supervftbl;
2861                                 s4         superindex;
2862
2863                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2864                                         super = NULL;
2865                                         superindex = 0;
2866                                         supervftbl = NULL;
2867                                 }
2868                                 else {
2869                                         super = iptr->sx.s23.s3.c.cls;
2870                                         superindex = super->index;
2871                                         supervftbl = super->vftbl;
2872                                 }
2873
2874 #if defined(ENABLE_THREADS)
2875                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
2876 #endif
2877                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2878
2879                                 /* calculate interface checkcast code size */
2880
2881                                 s2 = 3; /* mov_membase_reg */
2882                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
2883
2884                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
2885                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
2886                                         3 /* test */ + 6 /* jcc */;
2887
2888                                 if (super == NULL)
2889                                         s2 += (opt_shownops ? 5 : 0);
2890
2891                                 /* calculate class checkcast code size */
2892
2893                                 s3 = 3; /* mov_membase_reg */
2894                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
2895                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
2896
2897 #if 0
2898                                 if (s1 != REG_ITMP1) {
2899                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
2900                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
2901                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
2902                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
2903                                         a += 3;    /* sub */
2904
2905                                 } else
2906 #endif
2907                                         {
2908                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
2909                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
2910                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2911                                         }
2912
2913                                 s3 += 3 /* cmp */ + 6 /* jcc */;
2914
2915                                 if (super == NULL)
2916                                         s3 += (opt_shownops ? 5 : 0);
2917
2918                                 /* if class is not resolved, check which code to call */
2919
2920                                 if (super == NULL) {
2921                                         M_TEST(s1);
2922                                         M_BEQ(6 + (opt_shownops ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
2923
2924                                         codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2925                                                                                   iptr->sx.s23.s3.c.ref, 0);
2926
2927                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2928                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2929                                         M_BEQ(s2 + 5);
2930                                 }
2931
2932                                 /* interface checkcast code */
2933
2934                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2935                                         if (super != NULL) {
2936                                                 M_TEST(s1);
2937                                                 M_BEQ(s2);
2938                                         }
2939
2940                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2941
2942                                         if (super == NULL) {
2943                                                 codegen_add_patch_ref(cd,
2944                                                                                           PATCHER_checkcast_instanceof_interface,
2945                                                                                           iptr->sx.s23.s3.c.ref,
2946                                                                                           0);
2947                                         }
2948
2949                                         emit_movl_membase32_reg(cd, REG_ITMP2,
2950                                                                                           OFFSET(vftbl_t, interfacetablelength),
2951                                                                                           REG_ITMP3);
2952                                         /* XXX TWISTI: should this be int arithmetic? */
2953                                         M_LSUB_IMM32(superindex, REG_ITMP3);
2954                                         M_TEST(REG_ITMP3);
2955                                         M_BLE(0);
2956                                         codegen_add_classcastexception_ref(cd, s1);
2957                                         emit_mov_membase32_reg(cd, REG_ITMP2,
2958                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
2959                                                                                          superindex * sizeof(methodptr*),
2960                                                                                          REG_ITMP3);
2961                                         M_TEST(REG_ITMP3);
2962                                         M_BEQ(0);
2963                                         codegen_add_classcastexception_ref(cd, s1);
2964
2965                                         if (super == NULL)
2966                                                 M_JMP_IMM(s3);
2967                                 }
2968
2969                                 /* class checkcast code */
2970
2971                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2972                                         if (super != NULL) {
2973                                                 M_TEST(s1);
2974                                                 M_BEQ(s3);
2975                                         }
2976
2977                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2978
2979                                         if (super == NULL) {
2980                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_class,
2981                                                                                           iptr->sx.s23.s3.c.ref,
2982                                                                                           0);
2983                                         }
2984
2985                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2986 #if defined(ENABLE_THREADS)
2987                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
2988 #endif
2989                                         emit_movl_membase32_reg(cd, REG_ITMP2,
2990                                                                                           OFFSET(vftbl_t, baseval),
2991                                                                                           REG_ITMP2);
2992                                         /*                                      if (s1 != REG_ITMP1) { */
2993                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2994                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2995                                         /*                                                                                              REG_ITMP1); */
2996                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2997                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2998                                         /*                                                                                              REG_ITMP3); */
2999                                         /*  #if defined(ENABLE_THREADS) */
3000                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3001                                         /*  #endif */
3002                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
3003
3004                                         /*                                      } else { */
3005                                         emit_movl_membase32_reg(cd, REG_ITMP3,
3006                                                                                           OFFSET(vftbl_t, baseval),
3007                                                                                           REG_ITMP3);
3008                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3009                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3010                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3011                                         /*                                      } */
3012 #if defined(ENABLE_THREADS)
3013                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3014 #endif
3015                                         M_LCMP(REG_ITMP3, REG_ITMP2);
3016                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3017                                         codegen_add_classcastexception_ref(cd, s1);
3018                                 }
3019
3020                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
3021                         }
3022                         else {
3023                                 /* array type cast-check */
3024
3025                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
3026                                 M_INTMOVE(s1, REG_A0);
3027
3028                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3029                                         codegen_add_patch_ref(cd, PATCHER_builtin_arraycheckcast,
3030                                                                                   iptr->sx.s23.s3.c.ref, 0);
3031                                 }
3032
3033                                 M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
3034                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
3035                                 M_CALL(REG_ITMP1);
3036
3037                                 /* s1 may have been destroyed over the function call */
3038                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
3039                                 M_TEST(REG_RESULT);
3040                                 M_BEQ(0);
3041                                 codegen_add_classcastexception_ref(cd, s1);
3042
3043                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
3044                         }
3045
3046                         M_INTMOVE(s1, d);
3047                         emit_store_dst(jd, iptr, d);
3048                         break;
3049
3050                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3051
3052                                       /* val.a: (classinfo *) superclass              */
3053
3054                         /*  superclass is an interface:
3055                          *
3056                          *  return (sub != NULL) &&
3057                          *         (sub->vftbl->interfacetablelength > super->index) &&
3058                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3059                          *
3060                          *  superclass is a class:
3061                          *
3062                          *  return ((sub != NULL) && (0
3063                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3064                          *          super->vftbl->diffvall));
3065                          */
3066
3067                         {
3068                         classinfo *super;
3069                         vftbl_t   *supervftbl;
3070                         s4         superindex;
3071
3072                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3073                                 super = NULL;
3074                                 superindex = 0;
3075                                 supervftbl = NULL;
3076
3077                         } else {
3078                                 super = iptr->sx.s23.s3.c.cls;
3079                                 superindex = super->index;
3080                                 supervftbl = super->vftbl;
3081                         }
3082
3083 #if defined(ENABLE_THREADS)
3084             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3085 #endif
3086
3087                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
3088                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
3089                         if (s1 == d) {
3090                                 M_INTMOVE(s1, REG_ITMP1);
3091                                 s1 = REG_ITMP1;
3092                         }
3093
3094                         /* calculate interface instanceof code size */
3095
3096                         s2 = 3; /* mov_membase_reg */
3097                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3098                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3099                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3100                                 3 /* test */ + 4 /* setcc */;
3101
3102                         if (!super)
3103                                 s2 += (opt_shownops ? 5 : 0);
3104
3105                         /* calculate class instanceof code size */
3106
3107                         s3 = 3; /* mov_membase_reg */
3108                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3109                         s3 += 10; /* mov_imm_reg */
3110                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3111                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3112                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3113                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3114                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3115                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3116                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3117
3118                         if (super == NULL)
3119                                 s3 += (opt_shownops ? 5 : 0);
3120
3121                         emit_alu_reg_reg(cd, ALU_XOR, d, d);
3122
3123                         /* if class is not resolved, check which code to call */
3124
3125                         if (super == NULL) {
3126                                 emit_test_reg_reg(cd, s1, s1);
3127                                 emit_jcc(cd, CC_Z, (6 + (opt_shownops ? 5 : 0) +
3128                                                                                          7 + 6 + s2 + 5 + s3));
3129
3130                                 codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
3131                                                                           iptr->sx.s23.s3.c.ref, 0);
3132
3133                                 emit_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3134                                 emit_alul_imm_reg(cd, ALU_AND, ACC_INTERFACE, REG_ITMP3);
3135                                 emit_jcc(cd, CC_Z, s2 + 5);
3136                         }
3137
3138                         /* interface instanceof code */
3139
3140                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
3141                                 if (super != NULL) {
3142                                         emit_test_reg_reg(cd, s1, s1);
3143                                         emit_jcc(cd, CC_Z, s2);
3144                                 }
3145
3146                                 emit_mov_membase_reg(cd, s1,
3147                                                                          OFFSET(java_objectheader, vftbl),
3148                                                                          REG_ITMP1);
3149
3150                                 if (super == NULL) {
3151                                         codegen_add_patch_ref(cd,
3152                                                                                   PATCHER_checkcast_instanceof_interface,
3153                                                                                   iptr->sx.s23.s3.c.ref, 0);
3154                                 }
3155
3156                                 emit_movl_membase32_reg(cd, REG_ITMP1,
3157                                                                                   OFFSET(vftbl_t, interfacetablelength),
3158                                                                                   REG_ITMP3);
3159                                 emit_alu_imm32_reg(cd, ALU_SUB, superindex, REG_ITMP3);
3160                                 emit_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3161
3162                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3163
3164                                 emit_jcc(cd, CC_LE, a);
3165                                 emit_mov_membase32_reg(cd, REG_ITMP1,
3166                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3167                                                                                  superindex * sizeof(methodptr*),
3168                                                                                  REG_ITMP1);
3169                                 emit_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3170                                 emit_setcc_reg(cd, CC_NE, d);
3171
3172                                 if (!super)
3173                                         emit_jmp_imm(cd, s3);
3174                         }
3175
3176                         /* class instanceof code */
3177
3178                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
3179                                 if (super != NULL) {
3180                                         emit_test_reg_reg(cd, s1, s1);
3181                                         emit_jcc(cd, CC_E, s3);
3182                                 }
3183
3184                                 emit_mov_membase_reg(cd, s1,
3185                                                                            OFFSET(java_objectheader, vftbl),
3186                                                                            REG_ITMP1);
3187
3188                                 if (super == NULL) {
3189                                         codegen_add_patch_ref(cd, PATCHER_instanceof_class,
3190                                                                                   iptr->sx.s23.s3.c.ref, 0);
3191                                 }
3192
3193                                 emit_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3194 #if defined(ENABLE_THREADS)
3195                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3196 #endif
3197                                 emit_movl_membase_reg(cd, REG_ITMP1,
3198                                                                                 OFFSET(vftbl_t, baseval),
3199                                                                                 REG_ITMP1);
3200                                 emit_movl_membase_reg(cd, REG_ITMP2,
3201                                                                                 OFFSET(vftbl_t, diffval),
3202                                                                                 REG_ITMP3);
3203                                 emit_movl_membase_reg(cd, REG_ITMP2,
3204                                                                                 OFFSET(vftbl_t, baseval),
3205                                                                                 REG_ITMP2);
3206 #if defined(ENABLE_THREADS)
3207                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3208 #endif
3209                                 emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
3210                                 emit_alu_reg_reg(cd, ALU_XOR, d, d); /* may be REG_ITMP2 */
3211                                 emit_alu_reg_reg(cd, ALU_CMP, REG_ITMP3, REG_ITMP1);
3212                                 emit_setcc_reg(cd, CC_BE, d);
3213                         }
3214                         emit_store_dst(jd, iptr, d);
3215                         }
3216                         break;
3217
3218                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3219
3220                         /* check for negative sizes and copy sizes to stack if necessary  */
3221
3222                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
3223
3224                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
3225
3226                                 /* copy SAVEDVAR sizes to stack */
3227                                 var = VAR(iptr->sx.s23.s2.args[s1]);
3228
3229                                 /* Already Preallocated? */
3230                                 if (!(var->flags & PREALLOC)) {
3231                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
3232                                         M_LST(s2, REG_SP, s1 * 8);
3233                                 }
3234                         }
3235
3236                         /* is a patcher function set? */
3237
3238                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3239                                 codegen_add_patch_ref(cd, PATCHER_builtin_multianewarray,
3240                                                                           iptr->sx.s23.s3.c.ref, 0);
3241                         }
3242
3243                         /* a0 = dimension count */
3244
3245                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
3246
3247                         /* a1 = classinfo */
3248
3249                         M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
3250
3251                         /* a2 = pointer to dimensions = stack pointer */
3252
3253                         M_MOV(REG_SP, REG_A2);
3254
3255                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3256                         M_CALL(REG_ITMP1);
3257
3258                         /* check for exception before result assignment */
3259
3260                         M_TEST(REG_RESULT);
3261                         M_BEQ(0);
3262                         codegen_add_fillinstacktrace_ref(cd);
3263
3264                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
3265                         M_INTMOVE(REG_RESULT, s1);
3266                         emit_store_dst(jd, iptr, s1);
3267                         break;
3268
3269                 default:
3270                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3271                         return false;
3272         } /* switch */
3273
3274         } /* for instruction */
3275
3276         MCODECHECK(512); /* XXX require a lower number? */
3277
3278         /* At the end of a basic block we may have to append some nops,
3279            because the patcher stub calling code might be longer than the
3280            actual instruction. So codepatching does not change the
3281            following block unintentionally. */
3282
3283         if (cd->mcodeptr < cd->lastmcodeptr) {
3284                 while (cd->mcodeptr < cd->lastmcodeptr) {
3285                         M_NOP;
3286                 }
3287         }
3288
3289         } /* if (bptr -> flags >= BBREACHED) */
3290         } /* for basic block */
3291
3292         dseg_createlinenumbertable(cd);
3293
3294         /* generate stubs */
3295
3296         emit_exception_stubs(jd);
3297         emit_patcher_stubs(jd);
3298 #if 0
3299         emit_replacement_stubs(jd);
3300 #endif
3301
3302         codegen_finish(jd);
3303
3304         /* everything's ok */
3305
3306         return true;
3307 }
3308
3309
3310 /* createcompilerstub **********************************************************
3311
3312    Creates a stub routine which calls the compiler.
3313
3314 *******************************************************************************/
3315
3316 #define COMPILERSTUB_DATASIZE    3 * SIZEOF_VOID_P
3317 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
3318
3319 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
3320
3321
3322 u1 *createcompilerstub(methodinfo *m)
3323 {
3324         u1          *s;                     /* memory to hold the stub            */
3325         ptrint      *d;
3326         codeinfo    *code;
3327         codegendata *cd;
3328         s4           dumpsize;
3329
3330         s = CNEW(u1, COMPILERSTUB_SIZE);
3331
3332         /* set data pointer and code pointer */
3333
3334         d = (ptrint *) s;
3335         s = s + COMPILERSTUB_DATASIZE;
3336
3337         /* mark start of dump memory area */
3338
3339         dumpsize = dump_size();
3340
3341         cd = DNEW(codegendata);
3342         cd->mcodeptr = s;
3343
3344         /* Store the codeinfo pointer in the same place as in the
3345            methodheader for compiled methods. */
3346
3347         code = code_codeinfo_new(m);
3348
3349         d[0] = (ptrint) asm_call_jit_compiler;
3350         d[1] = (ptrint) m;
3351         d[2] = (ptrint) code;
3352
3353         /* code for the stub */
3354
3355         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
3356         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
3357         M_JMP(REG_ITMP3);
3358
3359 #if defined(ENABLE_STATISTICS)
3360         if (opt_stat)
3361                 count_cstub_len += COMPILERSTUB_SIZE;
3362 #endif
3363
3364         /* release dump area */
3365
3366         dump_release(dumpsize);
3367
3368         return s;
3369 }
3370
3371
3372 /* createnativestub ************************************************************
3373
3374    Creates a stub routine which calls a native method.
3375
3376 *******************************************************************************/
3377
3378 u1 *createnativestub(functionptr f, jitdata *jd, methoddesc *nmd)
3379 {
3380         methodinfo   *m;
3381         codeinfo     *code;
3382         codegendata  *cd;
3383         registerdata *rd;
3384         methoddesc   *md;
3385         s4            nativeparams;
3386         s4            i, j;                 /* count variables                    */
3387         s4            t;
3388         s4            s1, s2;
3389
3390         /* get required compiler data */
3391
3392         m    = jd->m;
3393         code = jd->code;
3394         cd   = jd->cd;
3395         rd   = jd->rd;
3396
3397         /* initialize variables */
3398
3399         md = m->parseddesc;
3400         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
3401
3402         /* calculate stack frame size */
3403
3404         cd->stackframesize =
3405                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
3406                 sizeof(localref_table) / SIZEOF_VOID_P +
3407                 INT_ARG_CNT + FLT_ARG_CNT +
3408                 1 +                       /* functionptr, TODO: store in data segment */
3409                 nmd->memuse;
3410
3411         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
3412
3413         /* create method header */
3414
3415         (void) dseg_addaddress(cd, code);                      /* CodeinfoPointer */
3416         (void) dseg_adds4(cd, cd->stackframesize * 8);         /* FrameSize       */
3417         (void) dseg_adds4(cd, 0);                              /* IsSync          */
3418         (void) dseg_adds4(cd, 0);                              /* IsLeaf          */
3419         (void) dseg_adds4(cd, 0);                              /* IntSave         */
3420         (void) dseg_adds4(cd, 0);                              /* FltSave         */
3421         (void) dseg_addlinenumbertablesize(cd);
3422         (void) dseg_adds4(cd, 0);                              /* ExTableSize     */
3423
3424         /* generate native method profiling code */
3425
3426         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3427                 /* count frequency */
3428
3429                 M_MOV_IMM(code, REG_ITMP3);
3430                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3431         }
3432
3433         /* generate stub code */
3434
3435         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3436
3437 #if !defined(NDEBUG)
3438         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3439                 emit_verbosecall_enter(jd);
3440 #endif
3441
3442         /* get function address (this must happen before the stackframeinfo) */
3443
3444 #if !defined(WITH_STATIC_CLASSPATH)
3445         if (f == NULL)
3446                 codegen_add_patch_ref(cd, PATCHER_resolve_native, m, 0);
3447 #endif
3448
3449         M_MOV_IMM(f, REG_ITMP3);
3450
3451
3452         /* save integer and float argument registers */
3453
3454         for (i = 0, j = 0; i < md->paramcount; i++) {
3455                 if (!md->params[i].inmemory) {
3456                         s1 = md->params[i].regoff;
3457
3458                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3459                                 M_LST(rd->argintregs[s1], REG_SP, j * 8);
3460                         else
3461                                 M_DST(rd->argfltregs[s1], REG_SP, j * 8);
3462
3463                         j++;
3464                 }
3465         }
3466
3467         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3468
3469         /* create dynamic stack info */
3470
3471         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3472         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3473         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3474         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3475         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3476         M_CALL(REG_ITMP1);
3477
3478         /* restore integer and float argument registers */
3479
3480         for (i = 0, j = 0; i < md->paramcount; i++) {
3481                 if (!md->params[i].inmemory) {
3482                         s1 = md->params[i].regoff;
3483
3484                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3485                                 M_LLD(rd->argintregs[s1], REG_SP, j * 8);
3486                         else
3487                                 M_DLD(rd->argfltregs[s1], REG_SP, j * 8);
3488
3489                         j++;
3490                 }
3491         }
3492
3493         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3494
3495
3496         /* copy or spill arguments to new locations */
3497
3498         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3499                 t = md->paramtypes[i].type;
3500
3501                 if (IS_INT_LNG_TYPE(t)) {
3502                         if (!md->params[i].inmemory) {
3503                                 s1 = rd->argintregs[md->params[i].regoff];
3504
3505                                 if (!nmd->params[j].inmemory) {
3506                                         s2 = rd->argintregs[nmd->params[j].regoff];
3507                                         M_INTMOVE(s1, s2);
3508
3509                                 } else {
3510                                         s2 = nmd->params[j].regoff;
3511                                         M_LST(s1, REG_SP, s2 * 8);
3512                                 }
3513
3514                         } else {
3515                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3516                                 s2 = nmd->params[j].regoff;
3517                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
3518                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
3519                         }
3520
3521                 } else {
3522                         /* We only copy spilled float arguments, as the float argument    */
3523                         /* registers keep unchanged.                                      */
3524
3525                         if (md->params[i].inmemory) {
3526                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3527                                 s2 = nmd->params[j].regoff;
3528
3529                                 if (IS_2_WORD_TYPE(t)) {
3530                                         M_DLD(REG_FTMP1, REG_SP, s1 * 8);
3531                                         M_DST(REG_FTMP1, REG_SP, s2 * 8);
3532                                 } else {
3533                                         M_FLD(REG_FTMP1, REG_SP, s1 * 8);
3534                                         M_FST(REG_FTMP1, REG_SP, s2 * 8);
3535                                 }
3536                         }
3537                 }
3538         }
3539
3540         /* put class into second argument register */
3541
3542         if (m->flags & ACC_STATIC)
3543                 M_MOV_IMM(m->class, REG_A1);
3544
3545         /* put env into first argument register */
3546
3547         M_MOV_IMM(_Jv_env, REG_A0);
3548
3549         /* do the native function call */
3550
3551         M_CALL(REG_ITMP3);
3552
3553         /* save return value */
3554
3555         if (md->returntype.type != TYPE_VOID) {
3556                 if (IS_INT_LNG_TYPE(md->returntype.type))
3557                         M_LST(REG_RESULT, REG_SP, 0 * 8);
3558                 else
3559                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
3560         }
3561
3562 #if !defined(NDEBUG)
3563         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3564                 emit_verbosecall_exit(jd);
3565 #endif
3566
3567         /* remove native stackframe info */
3568
3569         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3570         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3571         M_CALL(REG_ITMP1);
3572         M_MOV(REG_RESULT, REG_ITMP3);
3573
3574         /* restore return value */
3575
3576         if (md->returntype.type != TYPE_VOID) {
3577                 if (IS_INT_LNG_TYPE(md->returntype.type))
3578                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
3579                 else
3580                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3581         }
3582
3583         /* remove stackframe */
3584
3585         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3586
3587         /* test for exception */
3588
3589         M_TEST(REG_ITMP3);
3590         M_BNE(1);
3591         M_RET;
3592
3593         /* handle exception */
3594
3595         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3596         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3597         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3598
3599         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3600         M_JMP(REG_ITMP3);
3601
3602         /* generate patcher stubs */
3603
3604         emit_patcher_stubs(jd);
3605
3606         codegen_finish(jd);
3607
3608         return code->entrypoint;
3609 }
3610
3611
3612 /*
3613  * These are local overrides for various environment variables in Emacs.
3614  * Please do not remove this and leave it at the end of the file, where
3615  * Emacs will automagically detect them.
3616  * ---------------------------------------------------------------------
3617  * Local variables:
3618  * mode: c
3619  * indent-tabs-mode: t
3620  * c-basic-offset: 4
3621  * tab-width: 4
3622  * End:
3623  * vim:noexpandtab:sw=4:ts=4:
3624  */