src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    $Id: codegen.c 7918 2007-05-20 20:42:18Z michi $
  26
  27 */
  28
  29
  30 #include "config.h"
  31
  32 #include <assert.h>
  33 #include <stdio.h>
  34
  35 #include "vm/types.h"
  36
  37 #include "md-abi.h"
  38
  39 #include "vm/jit/x86_64/arch.h"
  40 #include "vm/jit/x86_64/codegen.h"
  41 #include "vm/jit/x86_64/emit.h"
  42
  43 #include "mm/memory.h"
  44
  45 #include "native/jni.h"
  46 #include "native/native.h"
  47
  48 #include "threads/lock-common.h"
  49
  50 #include "vm/builtin.h"
  51 #include "vm/exceptions.h"
  52 #include "vm/global.h"
  53 #include "vm/stringlocal.h"
  54 #include "vm/vm.h"
  55
  56 #include "vm/jit/abi.h"
  57 #include "vm/jit/asmpart.h"
  58 #include "vm/jit/codegen-common.h"
  59 #include "vm/jit/dseg.h"
  60 #include "vm/jit/emit-common.h"
  61 #include "vm/jit/jit.h"
  62 #include "vm/jit/methodheader.h"
  63 #include "vm/jit/parse.h"
  64 #include "vm/jit/patcher.h"
  65 #include "vm/jit/reg.h"
  66 #include "vm/jit/replace.h"
  67 #include "vm/jit/stacktrace.h"
  68
  69 #if defined(ENABLE_LSRA)
  70 # include "vm/jit/allocator/lsra.h"
  71 #endif
  72
  73 #include "vmcore/loader.h"
  74 #include "vmcore/options.h"
  75 #include "vmcore/statistics.h"
  76
  77
  78 /* codegen_emit ****************************************************************
  79
  80    Generates machine code.
  81
  82 *******************************************************************************/
  83
  84 bool codegen_emit(jitdata *jd)
  85 {
  86         methodinfo         *m;
  87         codeinfo           *code;
  88         codegendata        *cd;
  89         registerdata       *rd;
  90         s4                  len, s1, s2, s3, d, disp;
  91         u2                  currentline;
  92         ptrint              a;
  93         varinfo            *var, *dst;
  94         basicblock         *bptr;
  95         instruction        *iptr;
  96         exception_entry    *ex;
  97         constant_classref  *cr;
  98         unresolved_class   *uc;
  99         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 100         unresolved_method  *um;
 101         builtintable_entry *bte;
 102         methoddesc         *md;
 103         fieldinfo          *fi;
 104         unresolved_field   *uf;
 105         s4                  fieldtype;
 106         s4                 varindex;
 107
 108         /* get required compiler data */
 109
 110         m    = jd->m;
 111         code = jd->code;
 112         cd   = jd->cd;
 113         rd   = jd->rd;
 114
 115         /* prevent compiler warnings */
 116
 117         d   = 0;
 118         lm  = NULL;
 119         um  = NULL;
 120         bte = NULL;
 121
 122         {
 123         s4 i, p, t, l;
 124         s4 savedregs_num;
 125
 126         savedregs_num = 0;
 127
 128         /* space to save used callee saved registers */
 129
 130         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 131         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 132
 133         cd->stackframesize = rd->memuse + savedregs_num;
 134
 135 #if defined(ENABLE_THREADS)
 136         /* space to save argument of monitor_enter */
 137
 138         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 139                 cd->stackframesize++;
 140 #endif
 141
 142         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 143            native code e.g. libc or jni (alignment problems with
 144            movaps). */
 145
 146         if (!jd->isleafmethod || opt_verbosecall)
 147                 cd->stackframesize |= 0x1;
 148
 149         /* create method header */
 150
 151         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 152         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 153
 154 #if defined(ENABLE_THREADS)
 155         /* IsSync contains the offset relative to the stack pointer for the
 156            argument of monitor_exit used in the exception handler. Since the
 157            offset could be zero and give a wrong meaning of the flag it is
 158            offset by one.
 159         */
 160
 161         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 162                 (void) dseg_add_unique_s4(cd, (rd->memuse + 1) * 8); /* IsSync        */
 163         else
 164 #endif
 165                 (void) dseg_add_unique_s4(cd, 0);                  /* IsSync          */
 166
 167         (void) dseg_add_unique_s4(cd, jd->isleafmethod);               /* IsLeaf  */
 168         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 169         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 170
 171         (void) dseg_addlinenumbertablesize(cd);
 172
 173         (void) dseg_add_unique_s4(cd, jd->exceptiontablelength); /* ExTableSize   */
 174
 175         /* create exception table */
 176
 177         for (ex = jd->exceptiontable; ex != NULL; ex = ex->down) {
 178                 dseg_add_target(cd, ex->start);
 179                 dseg_add_target(cd, ex->end);
 180                 dseg_add_target(cd, ex->handler);
 181                 (void) dseg_add_unique_address(cd, ex->catchtype.any);
 182         }
 183
 184 #if defined(ENABLE_PROFILING)
 185         /* generate method profiling code */
 186
 187         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 188                 /* count frequency */
 189
 190                 M_MOV_IMM(code, REG_ITMP3);
 191                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 192
 193                 PROFILE_CYCLE_START;
 194         }
 195 #endif
 196
 197         /* create stack frame (if necessary) */
 198
 199         if (cd->stackframesize)
 200                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 201
 202         /* save used callee saved registers */
 203
 204         p = cd->stackframesize;
 205         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 206                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 207         }
 208         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 209                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 210         }
 211
 212         /* take arguments out of register or stack frame */
 213
 214         md = m->parseddesc;
 215
 216         for (p = 0, l = 0; p < md->paramcount; p++) {
 217                 t = md->paramtypes[p].type;
 218
 219                 varindex = jd->local_map[l * 5 + t];
 220
 221                 l++;
 222                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 223                         l++;
 224
 225                 if (varindex == UNUSED)
 226                         continue;
 227
 228                 var = VAR(varindex);
 229
 230                 s1 = md->params[p].regoff;
 231
 232                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 233                         if (!md->params[p].inmemory) {           /* register arguments    */
 234                                 if (!IS_INMEMORY(var->flags))
 235                                         M_INTMOVE(s1, var->vv.regoff);
 236                                 else
 237                                     M_LST(s1, REG_SP, var->vv.regoff * 8);
 238                         }
 239                         else {                                 /* stack arguments       */
 240                                 if (!IS_INMEMORY(var->flags))
 241                                         /* + 8 for return address */
 242                                         M_LLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 243                                 else
 244                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 245                         }
 246                 }
 247                 else {                                     /* floating args         */
 248                         if (!md->params[p].inmemory) {           /* register arguments    */
 249                                 if (!IS_INMEMORY(var->flags))
 250                                         M_FLTMOVE(s1, var->vv.regoff);
 251                                 else
 252                                         M_DST(s1, REG_SP, var->vv.regoff * 8);
 253                         }
 254                         else {                                 /* stack arguments       */
 255                                 if (!IS_INMEMORY(var->flags))
 256                                         M_DLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 257                                 else
 258                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 259                         }
 260                 }
 261         }
 262
 263         /* save monitorenter argument */
 264
 265 #if defined(ENABLE_THREADS)
 266         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 267                 /* stack offset for monitor argument */
 268
 269                 s1 = rd->memuse;
 270
 271                 if (opt_verbosecall) {
 272                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 273
 274                         for (p = 0; p < INT_ARG_CNT; p++)
 275                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 276
 277                         for (p = 0; p < FLT_ARG_CNT; p++)
 278                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 279
 280                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 281                 }
 282
 283                 /* decide which monitor enter function to call */
 284
 285                 if (m->flags & ACC_STATIC) {
 286                         M_MOV_IMM(&m->class->object.header, REG_A0);
 287                 }
 288                 else {
 289                         M_TEST(REG_A0);
 290                         M_BNE(8);
 291                         M_ALD_MEM(REG_A0, EXCEPTION_HARDWARE_NULLPOINTER);
 292                 }
 293
 294                 M_AST(REG_A0, REG_SP, s1 * 8);
 295                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 296                 M_CALL(REG_ITMP1);
 297
 298                 if (opt_verbosecall) {
 299                         for (p = 0; p < INT_ARG_CNT; p++)
 300                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 301
 302                         for (p = 0; p < FLT_ARG_CNT; p++)
 303                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 304
 305                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 306                 }
 307         }
 308 #endif
 309
 310 #if !defined(NDEBUG)
 311         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 312                 emit_verbosecall_enter(jd);
 313 #endif /* !defined(NDEBUG) */
 314
 315         }
 316
 317         /* end of header generation */
 318
 319         /* create replacement points */
 320
 321         REPLACEMENT_POINTS_INIT(cd, jd);
 322
 323         /* walk through all basic blocks */
 324
 325         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 326
 327                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 328
 329                 if (bptr->flags >= BBREACHED) {
 330
 331                 /* branch resolving */
 332
 333                 codegen_resolve_branchrefs(cd, bptr);
 334
 335                 /* handle replacement points */
 336
 337                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 338
 339                 /* copy interface registers to their destination */
 340
 341                 len = bptr->indepth;
 342                 MCODECHECK(512);
 343
 344 #if defined(ENABLE_PROFILING)
 345                 /* generate basicblock profiling code */
 346
 347                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 348                         /* count frequency */
 349
 350                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 351                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 352
 353                         /* if this is an exception handler, start profiling again */
 354
 355                         if (bptr->type == BBTYPE_EXH)
 356                                 PROFILE_CYCLE_START;
 357                 }
 358 #endif
 359
 360 #if defined(ENABLE_LSRA)
 361                 if (opt_lsra) {
 362                         while (len) {
 363                                 len--;
 364                                 src = bptr->invars[len];
 365                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 366                                         if (bptr->type == BBTYPE_EXH) {
 367 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 368                                                 if (!IS_INMEMORY(src->flags))
 369                                                         d= src->vv.regoff;
 370                                                 else
 371                                                         d=REG_ITMP1;
 372                                                 M_INTMOVE(REG_ITMP1, d);
 373                                                 emit_store(jd, NULL, src, d);
 374                                         }
 375                                 }
 376                         }
 377
 378                 } else {
 379 #endif
 380
 381                 while (len) {
 382                         len--;
 383                         var = VAR(bptr->invars[len]);
 384                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 385                                 if (bptr->type == BBTYPE_EXH) {
 386                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 387                                         M_INTMOVE(REG_ITMP1, d);
 388                                         emit_store(jd, NULL, var, d);
 389                                 }
 390                         }
 391                         else {
 392                                 assert((var->flags & INOUT));
 393                         }
 394                 }
 395 #if defined(ENABLE_LSRA)
 396                 }
 397 #endif
 398                 /* walk through all instructions */
 399
 400                 len = bptr->icount;
 401                 currentline = 0;
 402
 403                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 404                         if (iptr->line != currentline) {
 405                                 dseg_addlinenumber(cd, iptr->line);
 406                                 currentline = iptr->line;
 407                         }
 408
 409                         MCODECHECK(1024);                         /* 1KB should be enough */
 410
 411                 switch (iptr->opc) {
 412                 case ICMD_NOP:        /* ...  ==> ...                                 */
 413                 case ICMD_POP:        /* ..., value  ==> ...                          */
 414                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 415                         break;
 416
 417                 case ICMD_INLINE_START:
 418
 419                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 420                         break;
 421
 422                 case ICMD_INLINE_BODY:
 423
 424                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 425                         dseg_addlinenumber_inline_start(cd, iptr);
 426                         dseg_addlinenumber(cd, iptr->line);
 427                         break;
 428
 429                 case ICMD_INLINE_END:
 430
 431                         dseg_addlinenumber_inline_end(cd, iptr);
 432                         dseg_addlinenumber(cd, iptr->line);
 433                         break;
 434
 435                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 436
 437                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 438                         emit_nullpointer_check(cd, iptr, s1);
 439                         break;
 440
 441                 /* constant operations ************************************************/
 442
 443                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 444
 445                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 446                         ICONST(d, iptr->sx.val.i);
 447                         emit_store_dst(jd, iptr, d);
 448                         break;
 449
 450                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 451
 452                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 453                         LCONST(d, iptr->sx.val.l);
 454                         emit_store_dst(jd, iptr, d);
 455                         break;
 456
 457                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 458
 459                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 460                         disp = dseg_add_float(cd, iptr->sx.val.f);
 461                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 462                         emit_store_dst(jd, iptr, d);
 463                         break;
 464
 465                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 466
 467                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 468                         disp = dseg_add_double(cd, iptr->sx.val.d);
 469                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 470                         emit_store_dst(jd, iptr, d);
 471                         break;
 472
 473                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 474
 475                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 476
 477                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 478                                 cr = iptr->sx.val.c.ref;
 479
 480 /*                              PROFILE_CYCLE_STOP; */
 481
 482                                 codegen_add_patch_ref(cd, PATCHER_aconst, cr, 0);
 483
 484 /*                              PROFILE_CYCLE_START; */
 485
 486                                 M_MOV_IMM(NULL, d);
 487
 488                         } else {
 489                                 if (iptr->sx.val.anyptr == 0)
 490                                         M_CLR(d);
 491                                 else
 492                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 493                         }
 494                         emit_store_dst(jd, iptr, d);
 495                         break;
 496
 497
 498                 /* load/store/copy/move operations ************************************/
 499
 500                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 501                 case ICMD_ALOAD:      /* s1 = local variable                          */
 502                 case ICMD_LLOAD:
 503                 case ICMD_FLOAD:
 504                 case ICMD_DLOAD:
 505                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 506                 case ICMD_LSTORE:
 507                 case ICMD_FSTORE:
 508                 case ICMD_DSTORE:
 509                 case ICMD_COPY:
 510                 case ICMD_MOVE:
 511
 512                         emit_copy(jd, iptr);
 513                         break;
 514
 515                 case ICMD_ASTORE:
 516                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 517                                 emit_copy(jd, iptr);
 518                         break;
 519
 520                 /* integer operations *************************************************/
 521
 522                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 523
 524                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 525                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 526                         M_INTMOVE(s1, d);
 527                         M_INEG(d);
 528                         emit_store_dst(jd, iptr, d);
 529                         break;
 530
 531                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 532
 533                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 534                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 535                         M_INTMOVE(s1, d);
 536                         M_LNEG(d);
 537                         emit_store_dst(jd, iptr, d);
 538                         break;
 539
 540                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 541
 542                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 543                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 544                         M_ISEXT(s1, d);
 545                         emit_store_dst(jd, iptr, d);
 546                         break;
 547
 548                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 549
 550                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 551                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 552                         M_IMOV(s1, d);
 553                         emit_store_dst(jd, iptr, d);
 554                         break;
 555
 556                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 557
 558                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 559                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 560                         M_BSEXT(s1, d);
 561                         emit_store_dst(jd, iptr, d);
 562                         break;
 563
 564                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 565
 566                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 567                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 568                         M_CZEXT(s1, d);
 569                         emit_store_dst(jd, iptr, d);
 570                         break;
 571
 572                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 573
 574                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 575                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 576                         M_SSEXT(s1, d);
 577                         emit_store_dst(jd, iptr, d);
 578                         break;
 579
 580
 581                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 582
 583                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 584                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 585                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 586                         if (s2 == d)
 587                                 M_IADD(s1, d);
 588                         else {
 589                                 M_INTMOVE(s1, d);
 590                                 M_IADD(s2, d);
 591                         }
 592                         emit_store_dst(jd, iptr, d);
 593                         break;
 594
 595                 case ICMD_IINC:
 596                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 597                                       /* sx.val.i = constant                             */
 598
 599                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 600                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 601
 602                         /* Using inc and dec is not faster than add (tested with
 603                            sieve). */
 604
 605                         M_INTMOVE(s1, d);
 606                         M_IADD_IMM(iptr->sx.val.i, d);
 607                         emit_store_dst(jd, iptr, d);
 608                         break;
 609
 610                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 611
 612                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 613                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 614                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 615                         if (s2 == d)
 616                                 M_LADD(s1, d);
 617                         else {
 618                                 M_INTMOVE(s1, d);
 619                                 M_LADD(s2, d);
 620                         }
 621                         emit_store_dst(jd, iptr, d);
 622                         break;
 623
 624                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 625                                       /* sx.val.l = constant                             */
 626
 627                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 628                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 629                         M_INTMOVE(s1, d);
 630                         if (IS_IMM32(iptr->sx.val.l))
 631                                 M_LADD_IMM(iptr->sx.val.l, d);
 632                         else {
 633                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 634                                 M_LADD(REG_ITMP2, d);
 635                         }
 636                         emit_store_dst(jd, iptr, d);
 637                         break;
 638
 639                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 640
 641                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 642                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 643                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 644                         if (s2 == d) {
 645                                 M_INTMOVE(s1, REG_ITMP1);
 646                                 M_ISUB(s2, REG_ITMP1);
 647                                 M_INTMOVE(REG_ITMP1, d);
 648                         } else {
 649                                 M_INTMOVE(s1, d);
 650                                 M_ISUB(s2, d);
 651                         }
 652                         emit_store_dst(jd, iptr, d);
 653                         break;
 654
 655                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 656                                       /* sx.val.i = constant                             */
 657
 658                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 659                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 660                         M_INTMOVE(s1, d);
 661                         M_ISUB_IMM(iptr->sx.val.i, d);
 662                         emit_store_dst(jd, iptr, d);
 663                         break;
 664
 665                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 666
 667                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 668                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 669                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 670                         if (s2 == d) {
 671                                 M_INTMOVE(s1, REG_ITMP1);
 672                                 M_LSUB(s2, REG_ITMP1);
 673                                 M_INTMOVE(REG_ITMP1, d);
 674                         } else {
 675                                 M_INTMOVE(s1, d);
 676                                 M_LSUB(s2, d);
 677                         }
 678                         emit_store_dst(jd, iptr, d);
 679                         break;
 680
 681                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 682                                       /* sx.val.l = constant                             */
 683
 684                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 685                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 686                         M_INTMOVE(s1, d);
 687                         if (IS_IMM32(iptr->sx.val.l))
 688                                 M_LSUB_IMM(iptr->sx.val.l, d);
 689                         else {
 690                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 691                                 M_LSUB(REG_ITMP2, d);
 692                         }
 693                         emit_store_dst(jd, iptr, d);
 694                         break;
 695
 696                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 697
 698                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 699                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 700                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 701                         if (s2 == d)
 702                                 M_IMUL(s1, d);
 703                         else {
 704                                 M_INTMOVE(s1, d);
 705                                 M_IMUL(s2, d);
 706                         }
 707                         emit_store_dst(jd, iptr, d);
 708                         break;
 709
 710                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 711                                       /* sx.val.i = constant                             */
 712
 713                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 714                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 715                         if (iptr->sx.val.i == 2) {
 716                                 M_INTMOVE(s1, d);
 717                                 M_ISLL_IMM(1, d);
 718                         } else
 719                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 720                         emit_store_dst(jd, iptr, d);
 721                         break;
 722
 723                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 724
 725                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 726                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 727                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 728                         if (s2 == d)
 729                                 M_LMUL(s1, d);
 730                         else {
 731                                 M_INTMOVE(s1, d);
 732                                 M_LMUL(s2, d);
 733                         }
 734                         emit_store_dst(jd, iptr, d);
 735                         break;
 736
 737                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 738                                       /* sx.val.l = constant                             */
 739
 740                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 741                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 742                         if (IS_IMM32(iptr->sx.val.l))
 743                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 744                         else {
 745                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 746                                 M_INTMOVE(s1, d);
 747                                 M_LMUL(REG_ITMP2, d);
 748                         }
 749                         emit_store_dst(jd, iptr, d);
 750                         break;
 751
 752                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 753
 754                         s1 = emit_load_s1(jd, iptr, RAX);
 755                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 756                         d = codegen_reg_of_dst(jd, iptr, RAX);
 757
 758                         M_INTMOVE(s1, RAX);
 759                         M_INTMOVE(s2, REG_ITMP3);
 760                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 761
 762                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 763
 764                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 765                         M_BNE(4 + 6);
 766                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 767                         M_BEQ(1 + 3);                                          /* 6 bytes */
 768
 769                         emit_cltd(cd);                                         /* 1 byte  */
 770                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 771
 772                         M_INTMOVE(RAX, d);
 773                         emit_store_dst(jd, iptr, d);
 774                         dst = VAROP(iptr->dst);
 775                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 776                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 777                         break;
 778
 779                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 780
 781                         s1 = emit_load_s1(jd, iptr, RAX);
 782                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 783                         d = codegen_reg_of_dst(jd, iptr, RDX);
 784
 785                         M_INTMOVE(s1, RAX);
 786                         M_INTMOVE(s2, REG_ITMP3);
 787                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 788
 789                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 790
 791                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 792                         M_BNE(3 + 4 + 6);
 793                         M_CLR(RDX);                                            /* 3 bytes */
 794                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 795                         M_BEQ(1 + 3);                                          /* 6 bytes */
 796
 797                         emit_cltd(cd);                                         /* 1 byte  */
 798                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 799
 800                         M_INTMOVE(RDX, d);
 801                         emit_store_dst(jd, iptr, d);
 802                         dst = VAROP(iptr->dst);
 803                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 804                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 805                         break;
 806
 807                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 808                                       /* sx.val.i = constant                             */
 809
 810                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 811                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 812                         M_INTMOVE(s1, REG_ITMP1);
 813                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 814                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 815                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 816                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 817                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 818                         emit_store_dst(jd, iptr, d);
 819                         break;
 820
 821                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 822                                       /* sx.val.i = constant                             */
 823
 824                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 825                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 826                         M_INTMOVE(s1, REG_ITMP1);
 827                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 828                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 829                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 830                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 831                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 832                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 833                         emit_store_dst(jd, iptr, d);
 834                         break;
 835
 836
 837                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 838
 839                         s1 = emit_load_s1(jd, iptr, RAX);
 840                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 841                         d = codegen_reg_of_dst(jd, iptr, RAX);
 842
 843                         M_INTMOVE(s1, RAX);
 844                         M_INTMOVE(s2, REG_ITMP3);
 845                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 846
 847                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 848
 849                         /* check as described in jvm spec */
 850                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 851                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 852                         M_BNE(4 + 6);
 853                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 854                         M_BEQ(2 + 3);                                          /* 6 bytes */
 855
 856                         emit_cqto(cd);                                         /* 2 bytes */
 857                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 858
 859                         M_INTMOVE(RAX, d);
 860                         emit_store_dst(jd, iptr, d);
 861                         dst = VAROP(iptr->dst);
 862                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 863                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 864                         break;
 865
 866                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 867
 868                         s1 = emit_load_s1(jd, iptr, RAX);
 869                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 870                         d = codegen_reg_of_dst(jd, iptr, RDX);
 871
 872                         M_INTMOVE(s1, RAX);
 873                         M_INTMOVE(s2, REG_ITMP3);
 874                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 875
 876                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 877
 878                         /* check as described in jvm spec */
 879                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 880                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 881                         M_BNE(3 + 4 + 6);
 882                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 883                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 884                         M_BEQ(2 + 3);                                          /* 6 bytes */
 885
 886                         emit_cqto(cd);                                         /* 2 bytes */
 887                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 888
 889                         M_INTMOVE(RDX, d);
 890                         emit_store_dst(jd, iptr, d);
 891                         dst = VAROP(iptr->dst);
 892                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 893                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 894                         break;
 895
 896                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 897                                       /* sx.val.i = constant                             */
 898
 899                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 900                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 901                         M_INTMOVE(s1, REG_ITMP1);
 902                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 903                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 904                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 905                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 906                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 907                         emit_store_dst(jd, iptr, d);
 908                         break;
 909
 910                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 911                                       /* sx.val.l = constant                             */
 912
 913                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 914                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 915                         M_INTMOVE(s1, REG_ITMP1);
 916                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 917                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 918                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 919                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 920                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 921                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 922                         emit_store_dst(jd, iptr, d);
 923                         break;
 924
 925                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 926
 927                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 928                         emit_ishift(jd, SHIFT_SHL, iptr);
 929                         break;
 930
 931                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 932                                       /* sx.val.i = constant                             */
 933
 934                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 935                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 936                         M_INTMOVE(s1, d);
 937                         M_ISLL_IMM(iptr->sx.val.i, d);
 938                         emit_store_dst(jd, iptr, d);
 939                         break;
 940
 941                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 942
 943                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 944                         emit_ishift(jd, SHIFT_SAR, iptr);
 945                         break;
 946
 947                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 948                                       /* sx.val.i = constant                             */
 949
 950                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 951                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 952                         M_INTMOVE(s1, d);
 953                         M_ISRA_IMM(iptr->sx.val.i, d);
 954                         emit_store_dst(jd, iptr, d);
 955                         break;
 956
 957                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 958
 959                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 960                         emit_ishift(jd, SHIFT_SHR, iptr);
 961                         break;
 962
 963                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 964                                       /* sx.val.i = constant                             */
 965
 966                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 967                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 968                         M_INTMOVE(s1, d);
 969                         M_ISRL_IMM(iptr->sx.val.i, d);
 970                         emit_store_dst(jd, iptr, d);
 971                         break;
 972
 973                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 974
 975                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 976                         emit_lshift(jd, SHIFT_SHL, iptr);
 977                         break;
 978
 979         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 980                                           /* sx.val.i = constant                             */
 981
 982                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 983                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 984                         M_INTMOVE(s1, d);
 985                         M_LSLL_IMM(iptr->sx.val.i, d);
 986                         emit_store_dst(jd, iptr, d);
 987                         break;
 988
 989                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 990
 991                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 992                         emit_lshift(jd, SHIFT_SAR, iptr);
 993                         break;
 994
 995                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 996                                       /* sx.val.i = constant                             */
 997
 998                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 999                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1000                         M_INTMOVE(s1, d);
1001                         M_LSRA_IMM(iptr->sx.val.i, d);
1002                         emit_store_dst(jd, iptr, d);
1003                         break;
1004
1005                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1006
1007                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1008                         emit_lshift(jd, SHIFT_SHR, iptr);
1009                         break;
1010
1011                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1012                                       /* sx.val.l = constant                             */
1013
1014                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1015                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1016                         M_INTMOVE(s1, d);
1017                         M_LSRL_IMM(iptr->sx.val.i, d);
1018                         emit_store_dst(jd, iptr, d);
1019                         break;
1020
1021                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1022
1023                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1024                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1025                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1026                         if (s2 == d)
1027                                 M_IAND(s1, d);
1028                         else {
1029                                 M_INTMOVE(s1, d);
1030                                 M_IAND(s2, d);
1031                         }
1032                         emit_store_dst(jd, iptr, d);
1033                         break;
1034
1035                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1036                                       /* sx.val.i = constant                             */
1037
1038                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1039                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1040                         M_INTMOVE(s1, d);
1041                         M_IAND_IMM(iptr->sx.val.i, d);
1042                         emit_store_dst(jd, iptr, d);
1043                         break;
1044
1045                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1046
1047                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1048                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1049                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1050                         if (s2 == d)
1051                                 M_LAND(s1, d);
1052                         else {
1053                                 M_INTMOVE(s1, d);
1054                                 M_LAND(s2, d);
1055                         }
1056                         emit_store_dst(jd, iptr, d);
1057                         break;
1058
1059                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1060                                       /* sx.val.l = constant                             */
1061
1062                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1063                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1064                         M_INTMOVE(s1, d);
1065                         if (IS_IMM32(iptr->sx.val.l))
1066                                 M_LAND_IMM(iptr->sx.val.l, d);
1067                         else {
1068                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1069                                 M_LAND(REG_ITMP2, d);
1070                         }
1071                         emit_store_dst(jd, iptr, d);
1072                         break;
1073
1074                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1075
1076                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1077                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1078                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1079                         if (s2 == d)
1080                                 M_IOR(s1, d);
1081                         else {
1082                                 M_INTMOVE(s1, d);
1083                                 M_IOR(s2, d);
1084                         }
1085                         emit_store_dst(jd, iptr, d);
1086                         break;
1087
1088                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1089                                       /* sx.val.i = constant                             */
1090
1091                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1092                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1093                         M_INTMOVE(s1, d);
1094                         M_IOR_IMM(iptr->sx.val.i, d);
1095                         emit_store_dst(jd, iptr, d);
1096                         break;
1097
1098                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1099
1100                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1101                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1102                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1103                         if (s2 == d)
1104                                 M_LOR(s1, d);
1105                         else {
1106                                 M_INTMOVE(s1, d);
1107                                 M_LOR(s2, d);
1108                         }
1109                         emit_store_dst(jd, iptr, d);
1110                         break;
1111
1112                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1113                                       /* sx.val.l = constant                             */
1114
1115                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1116                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1117                         M_INTMOVE(s1, d);
1118                         if (IS_IMM32(iptr->sx.val.l))
1119                                 M_LOR_IMM(iptr->sx.val.l, d);
1120                         else {
1121                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1122                                 M_LOR(REG_ITMP2, d);
1123                         }
1124                         emit_store_dst(jd, iptr, d);
1125                         break;
1126
1127                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1128
1129                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1130                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1131                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1132                         if (s2 == d)
1133                                 M_IXOR(s1, d);
1134                         else {
1135                                 M_INTMOVE(s1, d);
1136                                 M_IXOR(s2, d);
1137                         }
1138                         emit_store_dst(jd, iptr, d);
1139                         break;
1140
1141                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1142                                       /* sx.val.i = constant                             */
1143
1144                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1145                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1146                         M_INTMOVE(s1, d);
1147                         M_IXOR_IMM(iptr->sx.val.i, d);
1148                         emit_store_dst(jd, iptr, d);
1149                         break;
1150
1151                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1152
1153                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1154                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1155                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1156                         if (s2 == d)
1157                                 M_LXOR(s1, d);
1158                         else {
1159                                 M_INTMOVE(s1, d);
1160                                 M_LXOR(s2, d);
1161                         }
1162                         emit_store_dst(jd, iptr, d);
1163                         break;
1164
1165                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1166                                       /* sx.val.l = constant                             */
1167
1168                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1169                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1170                         M_INTMOVE(s1, d);
1171                         if (IS_IMM32(iptr->sx.val.l))
1172                                 M_LXOR_IMM(iptr->sx.val.l, d);
1173                         else {
1174                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1175                                 M_LXOR(REG_ITMP2, d);
1176                         }
1177                         emit_store_dst(jd, iptr, d);
1178                         break;
1179
1180
1181                 /* floating operations ************************************************/
1182
1183                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1184
1185                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1186                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1187                         disp = dseg_add_s4(cd, 0x80000000);
1188                         M_FLTMOVE(s1, d);
1189                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1190                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1191                         emit_store_dst(jd, iptr, d);
1192                         break;
1193
1194                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1195
1196                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1197                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1198                         disp = dseg_add_s8(cd, 0x8000000000000000);
1199                         M_FLTMOVE(s1, d);
1200                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1201                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1202                         emit_store_dst(jd, iptr, d);
1203                         break;
1204
1205                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1206
1207                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1208                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1209                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1210                         if (s2 == d)
1211                                 M_FADD(s1, d);
1212                         else {
1213                                 M_FLTMOVE(s1, d);
1214                                 M_FADD(s2, d);
1215                         }
1216                         emit_store_dst(jd, iptr, d);
1217                         break;
1218
1219                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1220
1221                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1222                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1223                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1224                         if (s2 == d)
1225                                 M_DADD(s1, d);
1226                         else {
1227                                 M_FLTMOVE(s1, d);
1228                                 M_DADD(s2, d);
1229                         }
1230                         emit_store_dst(jd, iptr, d);
1231                         break;
1232
1233                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1234
1235                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1236                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1237                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1238                         if (s2 == d) {
1239                                 M_FLTMOVE(s2, REG_FTMP2);
1240                                 s2 = REG_FTMP2;
1241                         }
1242                         M_FLTMOVE(s1, d);
1243                         M_FSUB(s2, d);
1244                         emit_store_dst(jd, iptr, d);
1245                         break;
1246
1247                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1248
1249                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1250                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1251                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1252                         if (s2 == d) {
1253                                 M_FLTMOVE(s2, REG_FTMP2);
1254                                 s2 = REG_FTMP2;
1255                         }
1256                         M_FLTMOVE(s1, d);
1257                         M_DSUB(s2, d);
1258                         emit_store_dst(jd, iptr, d);
1259                         break;
1260
1261                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1262
1263                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1264                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1265                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1266                         if (s2 == d)
1267                                 M_FMUL(s1, d);
1268                         else {
1269                                 M_FLTMOVE(s1, d);
1270                                 M_FMUL(s2, d);
1271                         }
1272                         emit_store_dst(jd, iptr, d);
1273                         break;
1274
1275                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1276
1277                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1278                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1279                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1280                         if (s2 == d)
1281                                 M_DMUL(s1, d);
1282                         else {
1283                                 M_FLTMOVE(s1, d);
1284                                 M_DMUL(s2, d);
1285                         }
1286                         emit_store_dst(jd, iptr, d);
1287                         break;
1288
1289                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1290
1291                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1292                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1293                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1294                         if (s2 == d) {
1295                                 M_FLTMOVE(s2, REG_FTMP2);
1296                                 s2 = REG_FTMP2;
1297                         }
1298                         M_FLTMOVE(s1, d);
1299                         M_FDIV(s2, d);
1300                         emit_store_dst(jd, iptr, d);
1301                         break;
1302
1303                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1304
1305                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1306                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1307                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1308                         if (s2 == d) {
1309                                 M_FLTMOVE(s2, REG_FTMP2);
1310                                 s2 = REG_FTMP2;
1311                         }
1312                         M_FLTMOVE(s1, d);
1313                         M_DDIV(s2, d);
1314                         emit_store_dst(jd, iptr, d);
1315                         break;
1316
1317                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1318
1319                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1320                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1321                         M_CVTIF(s1, d);
1322                         emit_store_dst(jd, iptr, d);
1323                         break;
1324
1325                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1326
1327                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1328                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1329                         M_CVTID(s1, d);
1330                         emit_store_dst(jd, iptr, d);
1331                         break;
1332
1333                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1334
1335                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1336                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1337                         M_CVTLF(s1, d);
1338                         emit_store_dst(jd, iptr, d);
1339                         break;
1340
1341                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1342
1343                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1344                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1345                         M_CVTLD(s1, d);
1346                         emit_store_dst(jd, iptr, d);
1347                         break;
1348
1349                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1350
1351                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1352                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1353                         M_CVTFI(s1, d);
1354                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1355                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1356                                 ((REG_RESULT == d) ? 0 : 3);
1357                         M_BNE(disp);
1358                         M_FLTMOVE(s1, REG_FTMP1);
1359                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1360                         M_CALL(REG_ITMP2);
1361                         M_INTMOVE(REG_RESULT, d);
1362                         emit_store_dst(jd, iptr, d);
1363                         break;
1364
1365                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1366
1367                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1368                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1369                         M_CVTDI(s1, d);
1370                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1371                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1372                                 ((REG_RESULT == d) ? 0 : 3);
1373                         M_BNE(disp);
1374                         M_FLTMOVE(s1, REG_FTMP1);
1375                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1376                         M_CALL(REG_ITMP2);
1377                         M_INTMOVE(REG_RESULT, d);
1378                         emit_store_dst(jd, iptr, d);
1379                         break;
1380
1381                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1382
1383                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1384                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1385                         M_CVTFL(s1, d);
1386                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1387                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1388                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1389                                 ((REG_RESULT == d) ? 0 : 3);
1390                         M_BNE(disp);
1391                         M_FLTMOVE(s1, REG_FTMP1);
1392                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1393                         M_CALL(REG_ITMP2);
1394                         M_INTMOVE(REG_RESULT, d);
1395                         emit_store_dst(jd, iptr, d);
1396                         break;
1397
1398                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1399
1400                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1401                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1402                         M_CVTDL(s1, d);
1403                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1404                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1405                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1406                                 ((REG_RESULT == d) ? 0 : 3);
1407                         M_BNE(disp);
1408                         M_FLTMOVE(s1, REG_FTMP1);
1409                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1410                         M_CALL(REG_ITMP2);
1411                         M_INTMOVE(REG_RESULT, d);
1412                         emit_store_dst(jd, iptr, d);
1413                         break;
1414
1415                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1416
1417                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1418                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1419                         M_CVTFD(s1, d);
1420                         emit_store_dst(jd, iptr, d);
1421                         break;
1422
1423                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1424
1425                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1426                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1427                         M_CVTDF(s1, d);
1428                         emit_store_dst(jd, iptr, d);
1429                         break;
1430
1431                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1432                                           /* == => 0, < => 1, > => -1 */
1433
1434                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1435                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1436                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1437                         M_CLR(d);
1438                         M_MOV_IMM(1, REG_ITMP1);
1439                         M_MOV_IMM(-1, REG_ITMP2);
1440                         emit_ucomiss_reg_reg(cd, s1, s2);
1441                         M_CMOVULT(REG_ITMP1, d);
1442                         M_CMOVUGT(REG_ITMP2, d);
1443                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1444                         emit_store_dst(jd, iptr, d);
1445                         break;
1446
1447                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1448                                           /* == => 0, < => 1, > => -1 */
1449
1450                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1451                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1452                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1453                         M_CLR(d);
1454                         M_MOV_IMM(1, REG_ITMP1);
1455                         M_MOV_IMM(-1, REG_ITMP2);
1456                         emit_ucomiss_reg_reg(cd, s1, s2);
1457                         M_CMOVULT(REG_ITMP1, d);
1458                         M_CMOVUGT(REG_ITMP2, d);
1459                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1460                         emit_store_dst(jd, iptr, d);
1461                         break;
1462
1463                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1464                                           /* == => 0, < => 1, > => -1 */
1465
1466                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1467                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1468                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1469                         M_CLR(d);
1470                         M_MOV_IMM(1, REG_ITMP1);
1471                         M_MOV_IMM(-1, REG_ITMP2);
1472                         emit_ucomisd_reg_reg(cd, s1, s2);
1473                         M_CMOVULT(REG_ITMP1, d);
1474                         M_CMOVUGT(REG_ITMP2, d);
1475                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1476                         emit_store_dst(jd, iptr, d);
1477                         break;
1478
1479                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1480                                           /* == => 0, < => 1, > => -1 */
1481
1482                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1483                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1484                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1485                         M_CLR(d);
1486                         M_MOV_IMM(1, REG_ITMP1);
1487                         M_MOV_IMM(-1, REG_ITMP2);
1488                         emit_ucomisd_reg_reg(cd, s1, s2);
1489                         M_CMOVULT(REG_ITMP1, d);
1490                         M_CMOVUGT(REG_ITMP2, d);
1491                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1492                         emit_store_dst(jd, iptr, d);
1493                         break;
1494
1495
1496                 /* memory operations **************************************************/
1497
1498                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1499
1500                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1501                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1502                         /* implicit null-pointer check */
1503                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1504                         emit_store_dst(jd, iptr, d);
1505                         break;
1506
1507                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1508
1509                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1510                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1511                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1512                         /* implicit null-pointer check */
1513                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1514                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1515                         emit_store_dst(jd, iptr, d);
1516                         break;
1517
1518                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1519
1520                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1521                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1522                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1523                         /* implicit null-pointer check */
1524                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1525                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1526                         emit_store_dst(jd, iptr, d);
1527                         break;
1528
1529                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1530
1531                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1532                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1533                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1534                         /* implicit null-pointer check */
1535                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1536                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1537                         emit_store_dst(jd, iptr, d);
1538                         break;
1539
1540                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1541
1542                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1543                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1544                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1545                         /* implicit null-pointer check */
1546                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1547                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1548                         emit_store_dst(jd, iptr, d);
1549                         break;
1550
1551                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1552
1553                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1554                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1555                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1556                         /* implicit null-pointer check */
1557                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1558                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1559                         emit_store_dst(jd, iptr, d);
1560                         break;
1561
1562                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1563
1564                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1565                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1566                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1567                         /* implicit null-pointer check */
1568                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1569                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1570                         emit_store_dst(jd, iptr, d);
1571                         break;
1572
1573                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1574
1575                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1576                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1577                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1578                         /* implicit null-pointer check */
1579                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1580                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1581                         emit_store_dst(jd, iptr, d);
1582                         break;
1583
1584                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1585
1586                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1587                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1588                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1589                         /* implicit null-pointer check */
1590                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1591                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1592                         emit_store_dst(jd, iptr, d);
1593                         break;
1594
1595
1596                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1597
1598                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1599                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1600                         /* implicit null-pointer check */
1601                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1602                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1603                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1604                         break;
1605
1606                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1607
1608                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1609                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1610                         /* implicit null-pointer check */
1611                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1612                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1613                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1614                         break;
1615
1616                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1617
1618                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1619                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1620                         /* implicit null-pointer check */
1621                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1622                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1623                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1624                         break;
1625
1626                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1627
1628                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1629                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1630                         /* implicit null-pointer check */
1631                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1632                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1633                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
1634                         break;
1635
1636                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1637
1638                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1639                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1640                         /* implicit null-pointer check */
1641                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1642                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1643                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
1644                         break;
1645
1646                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1647
1648                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1649                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1650                         /* implicit null-pointer check */
1651                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1652                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1653                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
1654                         break;
1655
1656                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1657
1658                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1659                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1660                         /* implicit null-pointer check */
1661                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1662                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1663                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
1664                         break;
1665
1666                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1667
1668                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1669                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1670                         /* implicit null-pointer check */
1671                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1672                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1673
1674                         M_MOV(s1, REG_A0);
1675                         M_MOV(s3, REG_A1);
1676                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
1677                         M_CALL(REG_ITMP1);
1678                         emit_exception_check(cd, iptr);
1679
1680                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1681                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1682                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1683                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1684                         break;
1685
1686
1687                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1688
1689                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1690                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1691                         /* implicit null-pointer check */
1692                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1693                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1694                         break;
1695
1696                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1697
1698                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1699                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1700                         /* implicit null-pointer check */
1701                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1702                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray, data[0]), s1, s2, 1);
1703                         break;
1704
1705                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1706
1707                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1708                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1709                         /* implicit null-pointer check */
1710                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1711                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1712                         break;
1713
1714                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1715
1716                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1717                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1718                         /* implicit null-pointer check */
1719                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1720                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray, data[0]), s1, s2, 2);
1721                         break;
1722
1723                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1724
1725                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1726                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1727                         /* implicit null-pointer check */
1728                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1729
1730                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1731                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1732                         }
1733                         else {
1734                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1735                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
1736                         }
1737                         break;
1738
1739                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1740
1741                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1742                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1743                         /* implicit null-pointer check */
1744                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1745                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1746                         break;
1747
1748
1749                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1750
1751                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1752                                 uf        = iptr->sx.s23.s3.uf;
1753                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1754                                 disp      = dseg_add_unique_address(cd, NULL);
1755                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1756
1757                                 /* must be calculated before codegen_add_patch_ref */
1758
1759                                 if (opt_shownops)
1760                                         disp -= PATCHER_CALL_SIZE;
1761
1762 /*                              PROFILE_CYCLE_STOP; */
1763
1764                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1765
1766 /*                              PROFILE_CYCLE_START; */
1767                         }
1768                         else {
1769                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1770                                 fieldtype = fi->type;
1771                                 disp      = dseg_add_address(cd, &(fi->value));
1772                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1773
1774                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1775                                         PROFILE_CYCLE_STOP;
1776
1777                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1778
1779                                         PROFILE_CYCLE_START;
1780
1781                                         if (opt_shownops)
1782                                                 disp -= PATCHER_CALL_SIZE;
1783                                 }
1784                         }
1785
1786                         /* This approach is much faster than moving the field
1787                            address inline into a register. */
1788
1789                         M_ALD(REG_ITMP1, RIP, disp);
1790
1791                         switch (fieldtype) {
1792                         case TYPE_INT:
1793                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1794                                 M_ILD(d, REG_ITMP1, 0);
1795                                 break;
1796                         case TYPE_LNG:
1797                         case TYPE_ADR:
1798                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1799                                 M_LLD(d, REG_ITMP1, 0);
1800                                 break;
1801                         case TYPE_FLT:
1802                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1803                                 M_FLD(d, REG_ITMP1, 0);
1804                                 break;
1805                         case TYPE_DBL:
1806                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1807                                 M_DLD(d, REG_ITMP1, 0);
1808                                 break;
1809                         }
1810                         emit_store_dst(jd, iptr, d);
1811                         break;
1812
1813                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1814
1815                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1816                                 uf        = iptr->sx.s23.s3.uf;
1817                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1818                                 disp      = dseg_add_unique_address(cd, NULL);
1819                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1820
1821                                 /* must be calculated before codegen_add_patch_ref */
1822
1823                                 if (opt_shownops)
1824                                         disp -= PATCHER_CALL_SIZE;
1825
1826 /*                              PROFILE_CYCLE_STOP; */
1827
1828                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1829
1830 /*                              PROFILE_CYCLE_START; */
1831                         }
1832                         else {
1833                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1834                                 fieldtype = fi->type;
1835                                 disp      = dseg_add_address(cd, &(fi->value));
1836                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1837
1838                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1839                                         PROFILE_CYCLE_STOP;
1840
1841                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1842
1843                                         PROFILE_CYCLE_START;
1844
1845                                         if (opt_shownops)
1846                                                 disp -= PATCHER_CALL_SIZE;
1847                                 }
1848                         }
1849
1850                         /* This approach is much faster than moving the field
1851                            address inline into a register. */
1852
1853                         M_ALD(REG_ITMP1, RIP, disp);
1854
1855                         switch (fieldtype) {
1856                         case TYPE_INT:
1857                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1858                                 M_IST(s1, REG_ITMP1, 0);
1859                                 break;
1860                         case TYPE_LNG:
1861                         case TYPE_ADR:
1862                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1863                                 M_LST(s1, REG_ITMP1, 0);
1864                                 break;
1865                         case TYPE_FLT:
1866                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1867                                 M_FST(s1, REG_ITMP1, 0);
1868                                 break;
1869                         case TYPE_DBL:
1870                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1871                                 M_DST(s1, REG_ITMP1, 0);
1872                                 break;
1873                         }
1874                         break;
1875
1876                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1877                                           /* val = value (in current instruction)     */
1878                                           /* following NOP)                           */
1879
1880                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1881                                 uf        = iptr->sx.s23.s3.uf;
1882                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1883                                 disp      = dseg_add_unique_address(cd, NULL);
1884                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1885
1886                                 /* must be calculated before codegen_add_patch_ref */
1887
1888                                 if (opt_shownops)
1889                                         disp -= PATCHER_CALL_SIZE;
1890
1891 /*                              PROFILE_CYCLE_STOP; */
1892
1893                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1894
1895 /*                              PROFILE_CYCLE_START; */
1896                         }
1897                         else {
1898                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1899                                 fieldtype = fi->type;
1900                                 disp      = dseg_add_address(cd, &(fi->value));
1901                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1902
1903                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1904                                         PROFILE_CYCLE_STOP;
1905
1906                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1907
1908                                         PROFILE_CYCLE_START;
1909
1910                                         if (opt_shownops)
1911                                                 disp -= PATCHER_CALL_SIZE;
1912                                 }
1913                         }
1914
1915                         /* This approach is much faster than moving the field
1916                            address inline into a register. */
1917
1918                         M_ALD(REG_ITMP1, RIP, disp);
1919
1920                         switch (fieldtype) {
1921                         case TYPE_INT:
1922                         case TYPE_FLT:
1923                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1924                                 break;
1925                         case TYPE_LNG:
1926                         case TYPE_ADR:
1927                         case TYPE_DBL:
1928                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1929                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1930                                 else {
1931                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1932                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1933                                 }
1934                                 break;
1935                         }
1936                         break;
1937
1938                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1939
1940                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1941
1942                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1943                                 uf        = iptr->sx.s23.s3.uf;
1944                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1945                                 disp      = 0;
1946
1947 /*                              PROFILE_CYCLE_STOP; */
1948
1949                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1950
1951 /*                              PROFILE_CYCLE_START; */
1952                         }
1953                         else {
1954                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1955                                 fieldtype = fi->type;
1956                                 disp      = fi->offset;
1957                         }
1958
1959                         /* implicit null-pointer check */
1960                         switch (fieldtype) {
1961                         case TYPE_INT:
1962                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1963                                 M_ILD32(d, s1, disp);
1964                                 break;
1965                         case TYPE_LNG:
1966                         case TYPE_ADR:
1967                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1968                                 M_LLD32(d, s1, disp);
1969                                 break;
1970                         case TYPE_FLT:
1971                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1972                                 M_FLD32(d, s1, disp);
1973                                 break;
1974                         case TYPE_DBL:
1975                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1976                                 M_DLD32(d, s1, disp);
1977                                 break;
1978                         }
1979                         emit_store_dst(jd, iptr, d);
1980                         break;
1981
1982                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1983
1984                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1985                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1986
1987                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1988                                 uf        = iptr->sx.s23.s3.uf;
1989                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1990                                 disp      = 0;
1991
1992 /*                              PROFILE_CYCLE_STOP; */
1993
1994                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1995
1996 /*                              PROFILE_CYCLE_START; */
1997                         }
1998                         else {
1999                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2000                                 fieldtype = fi->type;
2001                                 disp      = fi->offset;
2002                         }
2003
2004                         /* implicit null-pointer check */
2005                         switch (fieldtype) {
2006                         case TYPE_INT:
2007                                 M_IST32(s2, s1, disp);
2008                                 break;
2009                         case TYPE_LNG:
2010                         case TYPE_ADR:
2011                                 M_LST32(s2, s1, disp);
2012                                 break;
2013                         case TYPE_FLT:
2014                                 M_FST32(s2, s1, disp);
2015                                 break;
2016                         case TYPE_DBL:
2017                                 M_DST32(s2, s1, disp);
2018                                 break;
2019                         }
2020                         break;
2021
2022                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2023                                           /* val = value (in current instruction)     */
2024                                           /* following NOP)                           */
2025
2026                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2027
2028                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2029                                 uf        = iptr->sx.s23.s3.uf;
2030                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2031                                 disp      = 0;
2032
2033 /*                              PROFILE_CYCLE_STOP; */
2034
2035                                 codegen_add_patch_ref(cd, PATCHER_putfieldconst, uf, 0);
2036
2037 /*                              PROFILE_CYCLE_START; */
2038                         }
2039                         else {
2040                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2041                                 fieldtype = fi->type;
2042                                 disp      = fi->offset;
2043                         }
2044
2045                         /* implicit null-pointer check */
2046                         switch (fieldtype) {
2047                         case TYPE_INT:
2048                         case TYPE_FLT:
2049                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2050                                 break;
2051                         case TYPE_LNG:
2052                         case TYPE_ADR:
2053                         case TYPE_DBL:
2054                                 /* XXX why no check for IS_IMM32? */
2055                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2056                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2057                                 break;
2058                         }
2059                         break;
2060
2061
2062                 /* branch operations **************************************************/
2063
2064                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2065
2066                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2067                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2068
2069                         PROFILE_CYCLE_STOP;
2070
2071 #ifdef ENABLE_VERIFIER
2072                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2073                                 uc = iptr->sx.s23.s2.uc;
2074
2075                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2076                         }
2077 #endif /* ENABLE_VERIFIER */
2078
2079                         M_CALL_IMM(0);                            /* passing exception pc */
2080                         M_POP(REG_ITMP2_XPC);
2081
2082                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2083                         M_JMP(REG_ITMP3);
2084                         break;
2085
2086                 case ICMD_GOTO:         /* ... ==> ...                                */
2087                 case ICMD_RET:
2088
2089                         emit_br(cd, iptr->dst.block);
2090                         ALIGNCODENOP;
2091                         break;
2092
2093                 case ICMD_JSR:          /* ... ==> ...                                */
2094
2095                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2096                         ALIGNCODENOP;
2097                         break;
2098
2099                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2100                 case ICMD_IFNONNULL:
2101
2102                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2103                         M_TEST(s1);
2104                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2105                         break;
2106
2107                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2108                 case ICMD_IFLT:
2109                 case ICMD_IFLE:
2110                 case ICMD_IFNE:
2111                 case ICMD_IFGT:
2112                 case ICMD_IFGE:
2113
2114                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2115                         M_ICMP_IMM(iptr->sx.val.i, s1);
2116                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2117                         break;
2118
2119                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2120                 case ICMD_IF_LNE:
2121                 case ICMD_IF_LLT:
2122                 case ICMD_IF_LGE:
2123                 case ICMD_IF_LGT:
2124                 case ICMD_IF_LLE:
2125
2126                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2127                         if (IS_IMM32(iptr->sx.val.l))
2128                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2129                         else {
2130                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2131                                 M_LCMP(REG_ITMP2, s1);
2132                         }
2133                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2134                         break;
2135
2136                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2137                 case ICMD_IF_ICMPNE:
2138                 case ICMD_IF_ICMPLT:
2139                 case ICMD_IF_ICMPGE:
2140                 case ICMD_IF_ICMPGT:
2141                 case ICMD_IF_ICMPLE:
2142
2143                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2144                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2145                         M_ICMP(s2, s1);
2146                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2147                         break;
2148
2149                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2150                 case ICMD_IF_ACMPNE:
2151
2152                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2153                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2154                         M_LCMP(s2, s1);
2155                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2156                         break;
2157
2158                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2159                 case ICMD_IF_LCMPNE:
2160                 case ICMD_IF_LCMPLT:
2161                 case ICMD_IF_LCMPGE:
2162                 case ICMD_IF_LCMPGT:
2163                 case ICMD_IF_LCMPLE:
2164
2165                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2166                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2167                         M_LCMP(s2, s1);
2168                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2169                         break;
2170
2171                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2172                 case ICMD_LRETURN:
2173
2174                         REPLACEMENT_POINT_RETURN(cd, iptr);
2175                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2176                         M_INTMOVE(s1, REG_RESULT);
2177                         goto nowperformreturn;
2178
2179                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2180
2181                         REPLACEMENT_POINT_RETURN(cd, iptr);
2182                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2183                         M_INTMOVE(s1, REG_RESULT);
2184
2185 #ifdef ENABLE_VERIFIER
2186                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2187                                 uc = iptr->sx.s23.s2.uc;
2188
2189                                 PROFILE_CYCLE_STOP;
2190
2191                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2192
2193                                 PROFILE_CYCLE_START;
2194                         }
2195 #endif /* ENABLE_VERIFIER */
2196                         goto nowperformreturn;
2197
2198                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2199                 case ICMD_DRETURN:
2200
2201                         REPLACEMENT_POINT_RETURN(cd, iptr);
2202                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2203                         M_FLTMOVE(s1, REG_FRESULT);
2204                         goto nowperformreturn;
2205
2206                 case ICMD_RETURN:      /* ...  ==> ...                                */
2207
2208                         REPLACEMENT_POINT_RETURN(cd, iptr);
2209
2210 nowperformreturn:
2211                         {
2212                         s4 i, p;
2213
2214                         p = cd->stackframesize;
2215
2216 #if !defined(NDEBUG)
2217                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2218                                 emit_verbosecall_exit(jd);
2219 #endif /* !defined(NDEBUG) */
2220
2221 #if defined(ENABLE_THREADS)
2222                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2223                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2224
2225                                 /* we need to save the proper return value */
2226                                 switch (iptr->opc) {
2227                                 case ICMD_IRETURN:
2228                                 case ICMD_ARETURN:
2229                                 case ICMD_LRETURN:
2230                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2231                                         break;
2232                                 case ICMD_FRETURN:
2233                                 case ICMD_DRETURN:
2234                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2235                                         break;
2236                                 }
2237
2238                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2239                                 M_CALL(REG_ITMP1);
2240
2241                                 /* and now restore the proper return value */
2242                                 switch (iptr->opc) {
2243                                 case ICMD_IRETURN:
2244                                 case ICMD_ARETURN:
2245                                 case ICMD_LRETURN:
2246                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2247                                         break;
2248                                 case ICMD_FRETURN:
2249                                 case ICMD_DRETURN:
2250                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2251                                         break;
2252                                 }
2253                         }
2254 #endif
2255
2256                         /* restore saved registers */
2257
2258                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2259                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2260                         }
2261                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2262                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2263                         }
2264
2265                         /* deallocate stack */
2266
2267                         if (cd->stackframesize)
2268                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2269
2270                         /* generate method profiling code */
2271
2272                         PROFILE_CYCLE_STOP;
2273
2274                         M_RET;
2275                         }
2276                         break;
2277
2278
2279                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2280                         {
2281                                 s4 i, l;
2282                                 branch_target_t *table;
2283
2284                                 table = iptr->dst.table;
2285
2286                                 l = iptr->sx.s23.s2.tablelow;
2287                                 i = iptr->sx.s23.s3.tablehigh;
2288
2289                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2290                                 M_INTMOVE(s1, REG_ITMP1);
2291
2292                                 if (l != 0)
2293                                         M_ISUB_IMM(l, REG_ITMP1);
2294
2295                                 /* number of targets */
2296                                 i = i - l + 1;
2297
2298                 /* range check */
2299
2300                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2301                                 emit_bugt(cd, table[0].block);
2302
2303                                 /* build jump table top down and use address of lowest entry */
2304
2305                                 table += i;
2306
2307                                 while (--i >= 0) {
2308                                         dseg_add_target(cd, table->block);
2309                                         --table;
2310                                 }
2311
2312                                 /* length of dataseg after last dseg_add_target is used
2313                                    by load */
2314
2315                                 M_MOV_IMM(0, REG_ITMP2);
2316                                 dseg_adddata(cd);
2317                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2318                                 M_JMP(REG_ITMP1);
2319                         }
2320                         break;
2321
2322
2323                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2324                         {
2325                                 s4 i;
2326                                 lookup_target_t *lookup;
2327
2328                                 lookup = iptr->dst.lookup;
2329
2330                                 i = iptr->sx.s23.s2.lookupcount;
2331
2332                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2333                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2334
2335                                 while (--i >= 0) {
2336                                         M_ICMP_IMM(lookup->value, s1);
2337                                         emit_beq(cd, lookup->target.block);
2338                                         lookup++;
2339                                 }
2340
2341                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2342                                 ALIGNCODENOP;
2343                         }
2344                         break;
2345
2346
2347                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2348
2349                         bte = iptr->sx.s23.s3.bte;
2350                         md  = bte->md;
2351                         goto gen_method;
2352
2353                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2354
2355                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2356                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2357                 case ICMD_INVOKEINTERFACE:
2358
2359                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2360
2361                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2362                                 lm = NULL;
2363                                 um = iptr->sx.s23.s3.um;
2364                                 md = um->methodref->parseddesc.md;
2365                         }
2366                         else {
2367                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2368                                 um = NULL;
2369                                 md = lm->parseddesc;
2370                         }
2371
2372 gen_method:
2373                         s3 = md->paramcount;
2374
2375                         MCODECHECK((20 * s3) + 128);
2376
2377                         /* copy arguments to registers or stack location */
2378
2379                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2380                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2381                                 d   = md->params[s3].regoff;
2382
2383                                 /* already preallocated (ARGVAR)? */
2384
2385                                 if (var->flags & PREALLOC)
2386                                         continue;
2387
2388                                 if (IS_INT_LNG_TYPE(var->type)) {
2389                                         if (!md->params[s3].inmemory) {
2390                                                 s1 = emit_load(jd, iptr, var, d);
2391                                                 M_INTMOVE(s1, d);
2392                                         }
2393                                         else {
2394                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2395                                                 M_LST(s1, REG_SP, d * 8);
2396                                         }
2397                                 }
2398                                 else {
2399                                         if (!md->params[s3].inmemory) {
2400                                                 s1 = emit_load(jd, iptr, var, d);
2401                                                 M_FLTMOVE(s1, d);
2402                                         }
2403                                         else {
2404                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2405
2406                                                 if (IS_2_WORD_TYPE(var->type))
2407                                                         M_DST(s1, REG_SP, d * 8);
2408                                                 else
2409                                                         M_FST(s1, REG_SP, d * 8);
2410                                         }
2411                                 }
2412                         }
2413
2414                         /* generate method profiling code */
2415
2416                         PROFILE_CYCLE_STOP;
2417
2418                         switch (iptr->opc) {
2419                         case ICMD_BUILTIN:
2420                                 if (bte->stub == NULL) {
2421                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2422                                 } else {
2423                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2424                                 }
2425                                 M_CALL(REG_ITMP1);
2426
2427                                 emit_exception_check(cd, iptr);
2428                                 break;
2429
2430                         case ICMD_INVOKESPECIAL:
2431                                 emit_nullpointer_check(cd, iptr, REG_A0);
2432                                 /* fall through */
2433
2434                         case ICMD_INVOKESTATIC:
2435                                 if (lm == NULL) {
2436                                         disp = dseg_add_unique_address(cd, NULL);
2437                                         disp = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
2438
2439                                         /* must be calculated before codegen_add_patch_ref */
2440
2441                                         if (opt_shownops)
2442                                                 disp -= PATCHER_CALL_SIZE;
2443
2444                                         codegen_add_patch_ref(cd, PATCHER_invokestatic_special,
2445                                                                                   um, disp);
2446
2447 /*                                      a = 0; */
2448                                 }
2449                                 else {
2450                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2451                                         disp = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
2452
2453 /*                                      a = (ptrint) lm->stubroutine; */
2454                                 }
2455
2456 /*                              M_MOV_IMM(a, REG_ITMP2); */
2457                                 M_ALD(REG_ITMP2, RIP, disp);
2458                                 M_CALL(REG_ITMP2);
2459                                 break;
2460
2461                         case ICMD_INVOKEVIRTUAL:
2462                                 if (lm == NULL) {
2463                                         codegen_add_patch_ref(cd, PATCHER_invokevirtual, um, 0);
2464
2465                                         s1 = 0;
2466                                 }
2467                                 else {
2468                                         s1 = OFFSET(vftbl_t, table[0]) +
2469                                                 sizeof(methodptr) * lm->vftblindex;
2470                                 }
2471
2472                                 /* implicit null-pointer check */
2473                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_objectheader, vftbl));
2474                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2475                                 M_CALL(REG_ITMP3);
2476                                 break;
2477
2478                         case ICMD_INVOKEINTERFACE:
2479                                 if (lm == NULL) {
2480                                         codegen_add_patch_ref(cd, PATCHER_invokeinterface, um, 0);
2481
2482                                         s1 = 0;
2483                                         s2 = 0;
2484                                 }
2485                                 else {
2486                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2487                                                 sizeof(methodptr) * lm->class->index;
2488
2489                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2490                                 }
2491
2492                                 /* implicit null-pointer check */
2493                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_objectheader, vftbl));
2494                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2495                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2496                                 M_CALL(REG_ITMP3);
2497                                 break;
2498                         }
2499
2500                         /* generate method profiling code */
2501
2502                         PROFILE_CYCLE_START;
2503
2504                         /* store size of call code in replacement point */
2505
2506                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2507
2508                         /* store return value */
2509
2510                         switch (md->returntype.type) {
2511                         case TYPE_INT:
2512                         case TYPE_LNG:
2513                         case TYPE_ADR:
2514                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2515                                 M_INTMOVE(REG_RESULT, s1);
2516                                 emit_store_dst(jd, iptr, s1);
2517                                 break;
2518                         case TYPE_FLT:
2519                         case TYPE_DBL:
2520                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2521                                 M_FLTMOVE(REG_FRESULT, s1);
2522                                 emit_store_dst(jd, iptr, s1);
2523                                 break;
2524                         default:
2525                                 /* TYPE_VOID */
2526                                 break;
2527                         }
2528                         break;
2529
2530
2531                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2532
2533                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2534                                 /* object type cast-check */
2535
2536                                 classinfo *super;
2537                                 vftbl_t   *supervftbl;
2538                                 s4         superindex;
2539
2540                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2541                                         super      = NULL;
2542                                         superindex = 0;
2543                                         supervftbl = NULL;
2544                                 }
2545                                 else {
2546                                         super      = iptr->sx.s23.s3.c.cls;
2547                                         superindex = super->index;
2548                                         supervftbl = super->vftbl;
2549                                 }
2550
2551                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2552                                         CODEGEN_CRITICAL_SECTION_NEW;
2553
2554                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2555
2556                                 /* if class is not resolved, check which code to call */
2557
2558                                 if (super == NULL) {
2559                                         M_TEST(s1);
2560                                         emit_label_beq(cd, BRANCH_LABEL_1);
2561
2562                                         codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2563                                                                                   iptr->sx.s23.s3.c.ref, 0);
2564
2565                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2566                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2567                                         emit_label_beq(cd, BRANCH_LABEL_2);
2568                                 }
2569
2570                                 /* interface checkcast code */
2571
2572                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2573                                         if (super != NULL) {
2574                                                 M_TEST(s1);
2575                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2576                                         }
2577
2578                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2579
2580                                         if (super == NULL) {
2581                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_interface,
2582                                                                                           iptr->sx.s23.s3.c.ref,
2583                                                                                           0);
2584                                         }
2585
2586                                         M_ILD32(REG_ITMP3,
2587                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2588                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2589                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2590
2591                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2592                                                         OFFSET(vftbl_t, interfacetable[0]) -
2593                                                         superindex * sizeof(methodptr*));
2594                                         M_TEST(REG_ITMP3);
2595                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2596
2597                                         if (super == NULL)
2598                                                 emit_label_br(cd, BRANCH_LABEL_4);
2599                                         else
2600                                                 emit_label(cd, BRANCH_LABEL_3);
2601                                 }
2602
2603                                 /* class checkcast code */
2604
2605                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2606                                         if (super == NULL) {
2607                                                 emit_label(cd, BRANCH_LABEL_2);
2608                                         }
2609                                         else {
2610                                                 M_TEST(s1);
2611                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2612                                         }
2613
2614                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2615
2616                                         if (super == NULL) {
2617                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_class,
2618                                                                                           iptr->sx.s23.s3.c.ref,
2619                                                                                           0);
2620                                         }
2621
2622                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2623
2624                                         CODEGEN_CRITICAL_SECTION_START;
2625
2626                                         M_ILD32(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2627
2628                                         /*                                      if (s1 != REG_ITMP1) { */
2629                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2630                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2631                                         /*                                                                                              REG_ITMP1); */
2632                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2633                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2634                                         /*                                                                                              REG_ITMP3); */
2635                                         /*  #if defined(ENABLE_THREADS) */
2636                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2637                                         /*  #endif */
2638                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2639
2640                                         /*                                      } else { */
2641
2642                                         M_ILD32(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
2643                                         M_ISUB(REG_ITMP3, REG_ITMP2);
2644                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2645                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2646                                         /*                                      } */
2647
2648                                         CODEGEN_CRITICAL_SECTION_END;
2649
2650                                         M_ICMP(REG_ITMP3, REG_ITMP2);
2651                                         emit_classcast_check(cd, iptr, BRANCH_UGT, REG_ITMP3, s1);
2652
2653                                         if (super != NULL)
2654                                                 emit_label(cd, BRANCH_LABEL_5);
2655                                 }
2656
2657                                 if (super == NULL) {
2658                                         emit_label(cd, BRANCH_LABEL_1);
2659                                         emit_label(cd, BRANCH_LABEL_4);
2660                                 }
2661
2662                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2663                         }
2664                         else {
2665                                 /* array type cast-check */
2666
2667                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2668                                 M_INTMOVE(s1, REG_A0);
2669
2670                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2671                                         codegen_add_patch_ref(cd, PATCHER_builtin_arraycheckcast,
2672                                                                                   iptr->sx.s23.s3.c.ref, 0);
2673                                 }
2674
2675                                 M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2676                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2677                                 M_CALL(REG_ITMP1);
2678
2679                                 /* s1 may have been destroyed over the function call */
2680                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2681                                 M_TEST(REG_RESULT);
2682                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2683
2684                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2685                         }
2686
2687                         M_INTMOVE(s1, d);
2688                         emit_store_dst(jd, iptr, d);
2689                         break;
2690
2691                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2692
2693                         {
2694                         classinfo *super;
2695                         vftbl_t   *supervftbl;
2696                         s4         superindex;
2697
2698                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2699                                 super      = NULL;
2700                                 superindex = 0;
2701                                 supervftbl = NULL;
2702                         }
2703                         else {
2704                                 super      = iptr->sx.s23.s3.c.cls;
2705                                 superindex = super->index;
2706                                 supervftbl = super->vftbl;
2707                         }
2708
2709                         if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2710                                 CODEGEN_CRITICAL_SECTION_NEW;
2711
2712                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2713                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2714
2715                         if (s1 == d) {
2716                                 M_INTMOVE(s1, REG_ITMP1);
2717                                 s1 = REG_ITMP1;
2718                         }
2719
2720                         M_CLR(d);
2721
2722                         /* if class is not resolved, check which code to call */
2723
2724                         if (super == NULL) {
2725                                 M_TEST(s1);
2726                                 emit_label_beq(cd, BRANCH_LABEL_1);
2727
2728                                 codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2729                                                                           iptr->sx.s23.s3.c.ref, 0);
2730
2731                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2732                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2733                                 emit_label_beq(cd, BRANCH_LABEL_2);
2734                         }
2735
2736                         /* interface instanceof code */
2737
2738                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2739                                 if (super != NULL) {
2740                                         M_TEST(s1);
2741                                         emit_label_beq(cd, BRANCH_LABEL_3);
2742                                 }
2743
2744                                 M_ALD(REG_ITMP1, s1, OFFSET(java_objectheader, vftbl));
2745
2746                                 if (super == NULL) {
2747                                         codegen_add_patch_ref(cd, PATCHER_instanceof_interface,
2748                                                                                   iptr->sx.s23.s3.c.ref, 0);
2749                                 }
2750
2751                                 M_ILD32(REG_ITMP3,
2752                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2753                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2754
2755                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2756
2757                                 M_BLE(a);
2758                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2759                                                 OFFSET(vftbl_t, interfacetable[0]) -
2760                                                 superindex * sizeof(methodptr*));
2761                                 M_TEST(REG_ITMP1);
2762                                 M_SETNE(d);
2763
2764                                 if (super == NULL)
2765                                         emit_label_br(cd, BRANCH_LABEL_4);
2766                                 else
2767                                         emit_label(cd, BRANCH_LABEL_3);
2768                         }
2769
2770                         /* class instanceof code */
2771
2772                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2773                                 if (super == NULL) {
2774                                         emit_label(cd, BRANCH_LABEL_2);
2775                                 }
2776                                 else {
2777                                         M_TEST(s1);
2778                                         emit_label_beq(cd, BRANCH_LABEL_5);
2779                                 }
2780
2781                                 M_ALD(REG_ITMP1, s1, OFFSET(java_objectheader, vftbl));
2782
2783                                 if (super == NULL) {
2784                                         codegen_add_patch_ref(cd, PATCHER_instanceof_class,
2785                                                                                   iptr->sx.s23.s3.c.ref, 0);
2786                                 }
2787
2788                                 M_MOV_IMM(supervftbl, REG_ITMP2);
2789
2790                                 CODEGEN_CRITICAL_SECTION_START;
2791
2792                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
2793                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2794                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2795
2796                                 CODEGEN_CRITICAL_SECTION_END;
2797
2798                                 M_ISUB(REG_ITMP2, REG_ITMP1);
2799                                 M_CLR(d); /* may be REG_ITMP2 */
2800                                 M_ICMP(REG_ITMP3, REG_ITMP1);
2801                                 M_SETULE(d);
2802
2803                                 if (super != NULL)
2804                                         emit_label(cd, BRANCH_LABEL_5);
2805                         }
2806
2807                         if (super == NULL) {
2808                                 emit_label(cd, BRANCH_LABEL_1);
2809                                 emit_label(cd, BRANCH_LABEL_4);
2810                         }
2811
2812                         emit_store_dst(jd, iptr, d);
2813                         }
2814                         break;
2815
2816                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2817
2818                         /* check for negative sizes and copy sizes to stack if necessary  */
2819
2820                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2821
2822                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2823
2824                                 /* copy SAVEDVAR sizes to stack */
2825                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2826
2827                                 /* Already Preallocated? */
2828                                 if (!(var->flags & PREALLOC)) {
2829                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2830                                         M_LST(s2, REG_SP, s1 * 8);
2831                                 }
2832                         }
2833
2834                         /* is a patcher function set? */
2835
2836                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2837                                 codegen_add_patch_ref(cd, PATCHER_builtin_multianewarray,
2838                                                                           iptr->sx.s23.s3.c.ref, 0);
2839                         }
2840
2841                         /* a0 = dimension count */
2842
2843                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2844
2845                         /* a1 = classinfo */
2846
2847                         M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2848
2849                         /* a2 = pointer to dimensions = stack pointer */
2850
2851                         M_MOV(REG_SP, REG_A2);
2852
2853                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2854                         M_CALL(REG_ITMP1);
2855
2856                         /* check for exception before result assignment */
2857
2858                         emit_exception_check(cd, iptr);
2859
2860                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2861                         M_INTMOVE(REG_RESULT, s1);
2862                         emit_store_dst(jd, iptr, s1);
2863                         break;
2864
2865                 default:
2866                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2867                                                                                    iptr->opc);
2868                         return false;
2869         } /* switch */
2870
2871         } /* for instruction */
2872
2873         MCODECHECK(512); /* XXX require a lower number? */
2874
2875         /* At the end of a basic block we may have to append some nops,
2876            because the patcher stub calling code might be longer than the
2877            actual instruction. So codepatching does not change the
2878            following block unintentionally. */
2879
2880         if (cd->mcodeptr < cd->lastmcodeptr) {
2881                 while (cd->mcodeptr < cd->lastmcodeptr) {
2882                         M_NOP;
2883                 }
2884         }
2885
2886         } /* if (bptr -> flags >= BBREACHED) */
2887         } /* for basic block */
2888
2889         dseg_createlinenumbertable(cd);
2890
2891         /* generate stubs */
2892
2893         emit_patcher_stubs(jd);
2894         REPLACEMENT_EMIT_STUBS(jd);
2895
2896         /* everything's ok */
2897
2898         return true;
2899 }
2900
2901
2902 /* codegen_emit_stub_compiler **************************************************
2903
2904    Emit a stub routine which calls the compiler.
2905
2906 *******************************************************************************/
2907
2908 void codegen_emit_stub_compiler(jitdata *jd)
2909 {
2910         methodinfo  *m;
2911         codegendata *cd;
2912
2913         /* get required compiler data */
2914
2915         m  = jd->m;
2916         cd = jd->cd;
2917
2918         /* code for the stub */
2919
2920         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
2921         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
2922         M_JMP(REG_ITMP3);
2923 }
2924
2925
2926 /* codegen_emit_stub_builtin ***************************************************
2927
2928    Creates a stub routine which calls a builtin function.
2929
2930 *******************************************************************************/
2931
2932 void codegen_emit_stub_builtin(jitdata *jd, builtintable_entry *bte)
2933 {
2934         codeinfo    *code;
2935         codegendata *cd;
2936         methoddesc  *md;
2937         s4           i, j;
2938         s4           s1, disp;
2939
2940         /* get required compiler data */
2941
2942         code = jd->code;
2943         cd   = jd->cd;
2944
2945         md = bte->md;
2946
2947         /* calculate stack frame size */
2948
2949         cd->stackframesize =
2950                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
2951                 1;                                                    /* return value */
2952
2953         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
2954
2955         /* create method header */
2956
2957         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2958         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2959         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
2960         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2961         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2962         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2963         (void) dseg_addlinenumbertablesize(cd);
2964         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
2965
2966         /* generate stub code */
2967
2968         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2969
2970 #if defined(ENABLE_GC_CACAO)
2971         /* Save callee saved integer registers in stackframeinfo (GC may
2972            need to recover them during a collection). */
2973
2974         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
2975                 OFFSET(stackframeinfo, intregs);
2976
2977         for (i = 0; i < INT_SAV_CNT; i++)
2978                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2979 #endif
2980
2981         /* save integer and float argument registers */
2982
2983         for (i = 0, j = 0; i < md->paramcount; i++) {
2984                 if (!md->params[i].inmemory) {
2985                         s1 = md->params[i].regoff;
2986
2987                         switch (md->paramtypes[i].type) {
2988                         case TYPE_INT:
2989                         case TYPE_LNG:
2990                         case TYPE_ADR:
2991                                 M_LST(s1, REG_SP, j * 8);
2992                                 break;
2993                         case TYPE_FLT:
2994                         case TYPE_DBL:
2995                                 M_DST(s1, REG_SP, j * 8);
2996                                 break;
2997                         }
2998
2999                         j++;
3000                 }
3001         }
3002
3003         /* create dynamic stack info */
3004
3005         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3006         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3007         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3008         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3009         M_MOV_IMM(codegen_stub_builtin_enter, REG_ITMP1);
3010         M_CALL(REG_ITMP1);
3011
3012         /* restore integer and float argument registers */
3013
3014         for (i = 0, j = 0; i < md->paramcount; i++) {
3015                 if (!md->params[i].inmemory) {
3016                         s1 = md->params[i].regoff;
3017
3018                         switch (md->paramtypes[i].type) {
3019                         case TYPE_INT:
3020                         case TYPE_LNG:
3021                         case TYPE_ADR:
3022                                 M_LLD(s1, REG_SP, j * 8);
3023                                 break;
3024                         case TYPE_FLT:
3025                         case TYPE_DBL:
3026                                 M_DLD(s1, REG_SP, j * 8);
3027                                 break;
3028                         }
3029
3030                         j++;
3031                 }
3032         }
3033
3034         /* call the builtin function */
3035
3036         M_MOV_IMM(bte->fp, REG_ITMP3);
3037         M_CALL(REG_ITMP3);
3038
3039         /* save return value */
3040
3041         switch (md->returntype.type) {
3042         case TYPE_INT:
3043         case TYPE_LNG:
3044         case TYPE_ADR:
3045                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3046                 break;
3047         case TYPE_FLT:
3048         case TYPE_DBL:
3049                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3050                 break;
3051         case TYPE_VOID:
3052                 break;
3053         }
3054
3055         /* remove native stackframe info */
3056
3057         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3058         M_MOV_IMM(codegen_stub_builtin_exit, REG_ITMP1);
3059         M_CALL(REG_ITMP1);
3060
3061         /* restore return value */
3062
3063         switch (md->returntype.type) {
3064         case TYPE_INT:
3065         case TYPE_LNG:
3066         case TYPE_ADR:
3067                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3068                 break;
3069         case TYPE_FLT:
3070         case TYPE_DBL:
3071                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3072                 break;
3073         case TYPE_VOID:
3074                 break;
3075         }
3076
3077 #if defined(ENABLE_GC_CACAO)
3078         /* Restore callee saved integer registers from stackframeinfo (GC
3079            might have modified them during a collection). */
3080
3081         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3082                 OFFSET(stackframeinfo, intregs);
3083
3084         for (i = 0; i < INT_SAV_CNT; i++)
3085                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3086 #endif
3087
3088         /* remove stackframe */
3089
3090         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3091         M_RET;
3092 }
3093
3094
3095 /* codegen_emit_stub_native ****************************************************
3096
3097    Emits a stub routine which calls a native method.
3098
3099 *******************************************************************************/
3100
3101 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f)
3102 {
3103         methodinfo  *m;
3104         codeinfo    *code;
3105         codegendata *cd;
3106         methoddesc  *md;
3107         s4           nativeparams;
3108         s4           i, j;
3109         s4           t;
3110         s4           s1, s2, disp;
3111
3112         /* get required compiler data */
3113
3114         m    = jd->m;
3115         code = jd->code;
3116         cd   = jd->cd;
3117
3118         /* initialize variables */
3119
3120         md = m->parseddesc;
3121         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
3122
3123         /* calculate stack frame size */
3124
3125         cd->stackframesize =
3126                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
3127                 sizeof(localref_table) / SIZEOF_VOID_P +
3128                 md->paramcount +
3129                 1 +                       /* functionptr, TODO: store in data segment */
3130                 nmd->memuse;
3131
3132         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
3133
3134         /* create method header */
3135
3136         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
3137         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
3138         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
3139         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
3140         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
3141         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
3142         (void) dseg_addlinenumbertablesize(cd);
3143         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
3144
3145 #if defined(ENABLE_PROFILING)
3146         /* generate native method profiling code */
3147
3148         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3149                 /* count frequency */
3150
3151                 M_MOV_IMM(code, REG_ITMP3);
3152                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3153         }
3154 #endif
3155
3156         /* generate stub code */
3157
3158         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3159
3160 #if !defined(NDEBUG)
3161         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3162                 emit_verbosecall_enter(jd);
3163 #endif
3164
3165         /* get function address (this must happen before the stackframeinfo) */
3166
3167 #if !defined(WITH_STATIC_CLASSPATH)
3168         if (f == NULL)
3169                 codegen_add_patch_ref(cd, PATCHER_resolve_native, m, 0);
3170 #endif
3171
3172         M_MOV_IMM(f, REG_ITMP3);
3173
3174 #if defined(ENABLE_GC_CACAO)
3175         /* Save callee saved integer registers in stackframeinfo (GC may
3176            need to recover them during a collection). */
3177
3178         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3179                 OFFSET(stackframeinfo, intregs);
3180
3181         for (i = 0; i < INT_SAV_CNT; i++)
3182                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3183 #endif
3184
3185         /* save integer and float argument registers */
3186
3187         for (i = 0; i < md->paramcount; i++) {
3188                 if (!md->params[i].inmemory) {
3189                         s1 = md->params[i].regoff;
3190
3191                         switch (md->paramtypes[i].type) {
3192                         case TYPE_INT:
3193                         case TYPE_LNG:
3194                         case TYPE_ADR:
3195                                 M_LST(s1, REG_SP, i * 8);
3196                                 break;
3197                         case TYPE_FLT:
3198                         case TYPE_DBL:
3199                                 M_DST(s1, REG_SP, i * 8);
3200                                 break;
3201                         }
3202                 }
3203         }
3204
3205         M_AST(REG_ITMP3, REG_SP, md->paramcount * 8);
3206
3207         /* create dynamic stack info */
3208
3209         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3210         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3211         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3212         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3213         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3214         M_CALL(REG_ITMP1);
3215
3216         /* restore integer and float argument registers */
3217
3218         for (i = 0; i < md->paramcount; i++) {
3219                 if (!md->params[i].inmemory) {
3220                         s1 = md->params[i].regoff;
3221
3222                         switch (md->paramtypes[i].type) {
3223                         case TYPE_INT:
3224                         case TYPE_LNG:
3225                         case TYPE_ADR:
3226                                 M_LLD(s1, REG_SP, i * 8);
3227                                 break;
3228                         case TYPE_FLT:
3229                         case TYPE_DBL:
3230                                 M_DLD(s1, REG_SP, i * 8);
3231                                 break;
3232                         }
3233                 }
3234         }
3235
3236         M_ALD(REG_ITMP3, REG_SP, md->paramcount * 8);
3237
3238         /* copy or spill arguments to new locations */
3239
3240         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3241                 t  = md->paramtypes[i].type;
3242                 s2 = nmd->params[j].regoff;
3243
3244                 if (IS_INT_LNG_TYPE(t)) {
3245                         if (!md->params[i].inmemory) {
3246                                 s1 = md->params[i].regoff;
3247
3248                                 if (!nmd->params[j].inmemory)
3249                                         M_INTMOVE(s1, s2);
3250                                 else
3251                                         M_LST(s1, REG_SP, s2 * 8);
3252                         }
3253                         else {
3254                                 s1 = md->params[i].regoff + cd->stackframesize + 1;/* +1 (RA) */
3255                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
3256                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
3257                         }
3258                 }
3259                 else {
3260                         /* We only copy spilled float arguments, as the float
3261                            argument registers keep unchanged. */
3262
3263                         if (md->params[i].inmemory) {
3264                                 s1 = md->params[i].regoff + cd->stackframesize + 1;/* +1 (RA) */
3265
3266                                 if (IS_2_WORD_TYPE(t)) {
3267                                         M_DLD(REG_FTMP1, REG_SP, s1 * 8);
3268                                         M_DST(REG_FTMP1, REG_SP, s2 * 8);
3269                                 }
3270                                 else {
3271                                         M_FLD(REG_FTMP1, REG_SP, s1 * 8);
3272                                         M_FST(REG_FTMP1, REG_SP, s2 * 8);
3273                                 }
3274                         }
3275                 }
3276         }
3277
3278         /* put class into second argument register */
3279
3280         if (m->flags & ACC_STATIC)
3281                 M_MOV_IMM(m->class, REG_A1);
3282
3283         /* put env into first argument register */
3284
3285         M_MOV_IMM(_Jv_env, REG_A0);
3286
3287         /* do the native function call */
3288
3289         M_CALL(REG_ITMP3);
3290
3291         /* save return value */
3292
3293         switch (md->returntype.type) {
3294         case TYPE_INT:
3295         case TYPE_LNG:
3296         case TYPE_ADR:
3297                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3298                 break;
3299         case TYPE_FLT:
3300         case TYPE_DBL:
3301                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3302                 break;
3303         case TYPE_VOID:
3304                 break;
3305         }
3306
3307 #if !defined(NDEBUG)
3308         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3309                 emit_verbosecall_exit(jd);
3310 #endif
3311
3312         /* remove native stackframe info */
3313
3314         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3315         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3316         M_CALL(REG_ITMP1);
3317         M_MOV(REG_RESULT, REG_ITMP3);
3318
3319         /* restore return value */
3320
3321         switch (md->returntype.type) {
3322         case TYPE_INT:
3323         case TYPE_LNG:
3324         case TYPE_ADR:
3325                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3326                 break;
3327         case TYPE_FLT:
3328         case TYPE_DBL:
3329                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3330                 break;
3331         case TYPE_VOID:
3332                 break;
3333         }
3334
3335 #if defined(ENABLE_GC_CACAO)
3336         /* Restore callee saved integer registers from stackframeinfo (GC
3337            might have modified them during a collection). */
3338
3339         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3340                 OFFSET(stackframeinfo, intregs);
3341
3342         for (i = 0; i < INT_SAV_CNT; i++)
3343                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3344 #endif
3345
3346         /* remove stackframe */
3347
3348         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3349
3350         /* test for exception */
3351
3352         M_TEST(REG_ITMP3);
3353         M_BNE(1);
3354         M_RET;
3355
3356         /* handle exception */
3357
3358         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3359         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3360         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3361
3362         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3363         M_JMP(REG_ITMP3);
3364
3365         /* generate patcher stubs */
3366
3367         emit_patcher_stubs(jd);
3368 }
3369
3370
3371 /*
3372  * These are local overrides for various environment variables in Emacs.
3373  * Please do not remove this and leave it at the end of the file, where
3374  * Emacs will automagically detect them.
3375  * ---------------------------------------------------------------------
3376  * Local variables:
3377  * mode: c
3378  * indent-tabs-mode: t
3379  * c-basic-offset: 4
3380  * tab-width: 4
3381  * End:
3382  * vim:noexpandtab:sw=4:ts=4:
3383  */