src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25 */
  26
  27
  28 #include "config.h"
  29
  30 #include <assert.h>
  31 #include <stdio.h>
  32
  33 #include "vm/types.h"
  34
  35 #include "md-abi.h"
  36
  37 #include "vm/jit/x86_64/arch.h"
  38 #include "vm/jit/x86_64/codegen.h"
  39 #include "vm/jit/x86_64/emit.h"
  40
  41 #include "mm/memory.h"
  42
  43 #include "native/jni.h"
  44 #include "native/localref.h"
  45 #include "native/native.h"
  46
  47 #include "threads/lock-common.h"
  48
  49 #include "vm/builtin.h"
  50 #include "vm/exceptions.h"
  51 #include "vm/global.h"
  52 #include "vm/stringlocal.h"
  53 #include "vm/vm.h"
  54
  55 #include "vm/jit/abi.h"
  56 #include "vm/jit/asmpart.h"
  57 #include "vm/jit/codegen-common.h"
  58 #include "vm/jit/dseg.h"
  59 #include "vm/jit/emit-common.h"
  60 #include "vm/jit/jit.h"
  61 #include "vm/jit/methodheader.h"
  62 #include "vm/jit/parse.h"
  63 #include "vm/jit/patcher.h"
  64 #include "vm/jit/reg.h"
  65 #include "vm/jit/replace.h"
  66 #include "vm/jit/stacktrace.h"
  67
  68 #if defined(ENABLE_LSRA)
  69 # include "vm/jit/allocator/lsra.h"
  70 #endif
  71
  72 #include "vmcore/loader.h"
  73 #include "vmcore/options.h"
  74 #include "vmcore/statistics.h"
  75
  76
  77 /* codegen_emit ****************************************************************
  78
  79    Generates machine code.
  80
  81 *******************************************************************************/
  82
  83 bool codegen_emit(jitdata *jd)
  84 {
  85         methodinfo         *m;
  86         codeinfo           *code;
  87         codegendata        *cd;
  88         registerdata       *rd;
  89         s4                  len, s1, s2, s3, d, disp;
  90         u2                  currentline;
  91         ptrint              a;
  92         varinfo            *var, *dst;
  93         basicblock         *bptr;
  94         instruction        *iptr;
  95         exception_entry    *ex;
  96         constant_classref  *cr;
  97         unresolved_class   *uc;
  98         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
  99         unresolved_method  *um;
 100         builtintable_entry *bte;
 101         methoddesc         *md;
 102         fieldinfo          *fi;
 103         unresolved_field   *uf;
 104         s4                  fieldtype;
 105         s4                 varindex;
 106
 107         /* get required compiler data */
 108
 109         m    = jd->m;
 110         code = jd->code;
 111         cd   = jd->cd;
 112         rd   = jd->rd;
 113
 114         /* prevent compiler warnings */
 115
 116         d   = 0;
 117         lm  = NULL;
 118         um  = NULL;
 119         bte = NULL;
 120
 121         {
 122         s4 i, p, t, l;
 123         s4 savedregs_num;
 124
 125         savedregs_num = 0;
 126
 127         /* space to save used callee saved registers */
 128
 129         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 130         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 131
 132         cd->stackframesize = rd->memuse + savedregs_num;
 133
 134 #if defined(ENABLE_THREADS)
 135         /* space to save argument of monitor_enter */
 136
 137         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 138                 cd->stackframesize++;
 139 #endif
 140
 141         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 142            native code e.g. libc or jni (alignment problems with
 143            movaps). */
 144
 145         if (!jd->isleafmethod || opt_verbosecall)
 146                 cd->stackframesize |= 0x1;
 147
 148         /* create method header */
 149
 150         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 151         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 152
 153 #if defined(ENABLE_THREADS)
 154         /* IsSync contains the offset relative to the stack pointer for the
 155            argument of monitor_exit used in the exception handler. Since the
 156            offset could be zero and give a wrong meaning of the flag it is
 157            offset by one.
 158         */
 159
 160         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 161                 (void) dseg_add_unique_s4(cd, (rd->memuse + 1) * 8); /* IsSync        */
 162         else
 163 #endif
 164                 (void) dseg_add_unique_s4(cd, 0);                  /* IsSync          */
 165
 166         (void) dseg_add_unique_s4(cd, jd->isleafmethod);               /* IsLeaf  */
 167         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 168         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 169
 170         (void) dseg_addlinenumbertablesize(cd);
 171
 172         (void) dseg_add_unique_s4(cd, jd->exceptiontablelength); /* ExTableSize   */
 173
 174         /* create exception table */
 175
 176         for (ex = jd->exceptiontable; ex != NULL; ex = ex->down) {
 177                 dseg_add_target(cd, ex->start);
 178                 dseg_add_target(cd, ex->end);
 179                 dseg_add_target(cd, ex->handler);
 180                 (void) dseg_add_unique_address(cd, ex->catchtype.any);
 181         }
 182
 183 #if defined(ENABLE_PROFILING)
 184         /* generate method profiling code */
 185
 186         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 187                 /* count frequency */
 188
 189                 M_MOV_IMM(code, REG_ITMP3);
 190                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 191
 192                 PROFILE_CYCLE_START;
 193         }
 194 #endif
 195
 196         /* create stack frame (if necessary) */
 197
 198         if (cd->stackframesize)
 199                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 200
 201         /* save used callee saved registers */
 202
 203         p = cd->stackframesize;
 204         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 205                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 206         }
 207         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 208                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 209         }
 210
 211         /* take arguments out of register or stack frame */
 212
 213         md = m->parseddesc;
 214
 215         for (p = 0, l = 0; p < md->paramcount; p++) {
 216                 t = md->paramtypes[p].type;
 217
 218                 varindex = jd->local_map[l * 5 + t];
 219
 220                 l++;
 221                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 222                         l++;
 223
 224                 if (varindex == UNUSED)
 225                         continue;
 226
 227                 var = VAR(varindex);
 228
 229                 s1 = md->params[p].regoff;
 230
 231                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 232                         if (!md->params[p].inmemory) {           /* register arguments    */
 233                                 if (!IS_INMEMORY(var->flags))
 234                                         M_INTMOVE(s1, var->vv.regoff);
 235                                 else
 236                                     M_LST(s1, REG_SP, var->vv.regoff);
 237                         }
 238                         else {                                 /* stack arguments       */
 239                                 if (!IS_INMEMORY(var->flags))
 240                                         /* + 8 for return address */
 241                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 242                                 else
 243                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 244                         }
 245                 }
 246                 else {                                     /* floating args         */
 247                         if (!md->params[p].inmemory) {           /* register arguments    */
 248                                 if (!IS_INMEMORY(var->flags))
 249                                         M_FLTMOVE(s1, var->vv.regoff);
 250                                 else
 251                                         M_DST(s1, REG_SP, var->vv.regoff);
 252                         }
 253                         else {                                 /* stack arguments       */
 254                                 if (!IS_INMEMORY(var->flags))
 255                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 256                                 else
 257                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 258                         }
 259                 }
 260         }
 261
 262         /* save monitorenter argument */
 263
 264 #if defined(ENABLE_THREADS)
 265         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 266                 /* stack offset for monitor argument */
 267
 268                 s1 = rd->memuse;
 269
 270                 if (opt_verbosecall) {
 271                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 272
 273                         for (p = 0; p < INT_ARG_CNT; p++)
 274                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 275
 276                         for (p = 0; p < FLT_ARG_CNT; p++)
 277                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 278
 279                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 280                 }
 281
 282                 /* decide which monitor enter function to call */
 283
 284                 if (m->flags & ACC_STATIC) {
 285                         M_MOV_IMM(&m->class->object.header, REG_A0);
 286                 }
 287                 else {
 288                         M_TEST(REG_A0);
 289                         M_BNE(8);
 290                         M_ALD_MEM(REG_A0, EXCEPTION_HARDWARE_NULLPOINTER);
 291                 }
 292
 293                 M_AST(REG_A0, REG_SP, s1 * 8);
 294                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 295                 M_CALL(REG_ITMP1);
 296
 297                 if (opt_verbosecall) {
 298                         for (p = 0; p < INT_ARG_CNT; p++)
 299                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 300
 301                         for (p = 0; p < FLT_ARG_CNT; p++)
 302                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 303
 304                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 305                 }
 306         }
 307 #endif
 308
 309 #if !defined(NDEBUG)
 310         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 311                 emit_verbosecall_enter(jd);
 312 #endif /* !defined(NDEBUG) */
 313
 314         }
 315
 316         /* end of header generation */
 317
 318         /* create replacement points */
 319
 320         REPLACEMENT_POINTS_INIT(cd, jd);
 321
 322         /* walk through all basic blocks */
 323
 324         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 325
 326                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 327
 328                 if (bptr->flags >= BBREACHED) {
 329
 330                 /* branch resolving */
 331
 332                 codegen_resolve_branchrefs(cd, bptr);
 333
 334                 /* handle replacement points */
 335
 336                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 337
 338                 /* copy interface registers to their destination */
 339
 340                 len = bptr->indepth;
 341                 MCODECHECK(512);
 342
 343 #if defined(ENABLE_PROFILING)
 344                 /* generate basicblock profiling code */
 345
 346                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 347                         /* count frequency */
 348
 349                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 350                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 351
 352                         /* if this is an exception handler, start profiling again */
 353
 354                         if (bptr->type == BBTYPE_EXH)
 355                                 PROFILE_CYCLE_START;
 356                 }
 357 #endif
 358
 359 #if defined(ENABLE_LSRA)
 360                 if (opt_lsra) {
 361                         while (len) {
 362                                 len--;
 363                                 src = bptr->invars[len];
 364                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 365                                         if (bptr->type == BBTYPE_EXH) {
 366 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 367                                                 if (!IS_INMEMORY(src->flags))
 368                                                         d= src->vv.regoff;
 369                                                 else
 370                                                         d=REG_ITMP1;
 371                                                 M_INTMOVE(REG_ITMP1, d);
 372                                                 emit_store(jd, NULL, src, d);
 373                                         }
 374                                 }
 375                         }
 376
 377                 } else {
 378 #endif
 379
 380                 while (len) {
 381                         len--;
 382                         var = VAR(bptr->invars[len]);
 383                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 384                                 if (bptr->type == BBTYPE_EXH) {
 385                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 386                                         M_INTMOVE(REG_ITMP1, d);
 387                                         emit_store(jd, NULL, var, d);
 388                                 }
 389                         }
 390                         else {
 391                                 assert((var->flags & INOUT));
 392                         }
 393                 }
 394 #if defined(ENABLE_LSRA)
 395                 }
 396 #endif
 397                 /* walk through all instructions */
 398
 399                 len = bptr->icount;
 400                 currentline = 0;
 401
 402                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 403                         if (iptr->line != currentline) {
 404                                 dseg_addlinenumber(cd, iptr->line);
 405                                 currentline = iptr->line;
 406                         }
 407
 408                         MCODECHECK(1024);                         /* 1KB should be enough */
 409
 410                 switch (iptr->opc) {
 411                 case ICMD_NOP:        /* ...  ==> ...                                 */
 412                 case ICMD_POP:        /* ..., value  ==> ...                          */
 413                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 414                         break;
 415
 416                 case ICMD_INLINE_START:
 417
 418                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 419                         break;
 420
 421                 case ICMD_INLINE_BODY:
 422
 423                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 424                         dseg_addlinenumber_inline_start(cd, iptr);
 425                         dseg_addlinenumber(cd, iptr->line);
 426                         break;
 427
 428                 case ICMD_INLINE_END:
 429
 430                         dseg_addlinenumber_inline_end(cd, iptr);
 431                         dseg_addlinenumber(cd, iptr->line);
 432                         break;
 433
 434                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 435
 436                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 437                         emit_nullpointer_check(cd, iptr, s1);
 438                         break;
 439
 440                 /* constant operations ************************************************/
 441
 442                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 443
 444                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 445                         ICONST(d, iptr->sx.val.i);
 446                         emit_store_dst(jd, iptr, d);
 447                         break;
 448
 449                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 450
 451                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 452                         LCONST(d, iptr->sx.val.l);
 453                         emit_store_dst(jd, iptr, d);
 454                         break;
 455
 456                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 457
 458                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 459                         disp = dseg_add_float(cd, iptr->sx.val.f);
 460                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 461                         emit_store_dst(jd, iptr, d);
 462                         break;
 463
 464                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 465
 466                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 467                         disp = dseg_add_double(cd, iptr->sx.val.d);
 468                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 469                         emit_store_dst(jd, iptr, d);
 470                         break;
 471
 472                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 473
 474                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 475
 476                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 477                                 cr = iptr->sx.val.c.ref;
 478
 479 /*                              PROFILE_CYCLE_STOP; */
 480
 481                                 codegen_add_patch_ref(cd, PATCHER_aconst, cr, 0);
 482
 483 /*                              PROFILE_CYCLE_START; */
 484
 485                                 M_MOV_IMM(NULL, d);
 486
 487                         } else {
 488                                 if (iptr->sx.val.anyptr == 0)
 489                                         M_CLR(d);
 490                                 else
 491                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 492                         }
 493                         emit_store_dst(jd, iptr, d);
 494                         break;
 495
 496
 497                 /* load/store/copy/move operations ************************************/
 498
 499                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 500                 case ICMD_ALOAD:      /* s1 = local variable                          */
 501                 case ICMD_LLOAD:
 502                 case ICMD_FLOAD:
 503                 case ICMD_DLOAD:
 504                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 505                 case ICMD_LSTORE:
 506                 case ICMD_FSTORE:
 507                 case ICMD_DSTORE:
 508                 case ICMD_COPY:
 509                 case ICMD_MOVE:
 510
 511                         emit_copy(jd, iptr);
 512                         break;
 513
 514                 case ICMD_ASTORE:
 515                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 516                                 emit_copy(jd, iptr);
 517                         break;
 518
 519                 /* integer operations *************************************************/
 520
 521                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 522
 523                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 524                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 525                         M_INTMOVE(s1, d);
 526                         M_INEG(d);
 527                         emit_store_dst(jd, iptr, d);
 528                         break;
 529
 530                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 531
 532                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 533                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 534                         M_INTMOVE(s1, d);
 535                         M_LNEG(d);
 536                         emit_store_dst(jd, iptr, d);
 537                         break;
 538
 539                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 540
 541                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 542                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 543                         M_ISEXT(s1, d);
 544                         emit_store_dst(jd, iptr, d);
 545                         break;
 546
 547                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 548
 549                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 550                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 551                         M_IMOV(s1, d);
 552                         emit_store_dst(jd, iptr, d);
 553                         break;
 554
 555                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 556
 557                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 558                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 559                         M_BSEXT(s1, d);
 560                         emit_store_dst(jd, iptr, d);
 561                         break;
 562
 563                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 564
 565                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 566                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 567                         M_CZEXT(s1, d);
 568                         emit_store_dst(jd, iptr, d);
 569                         break;
 570
 571                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 572
 573                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 574                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 575                         M_SSEXT(s1, d);
 576                         emit_store_dst(jd, iptr, d);
 577                         break;
 578
 579
 580                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 581
 582                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 583                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 584                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 585                         if (s2 == d)
 586                                 M_IADD(s1, d);
 587                         else {
 588                                 M_INTMOVE(s1, d);
 589                                 M_IADD(s2, d);
 590                         }
 591                         emit_store_dst(jd, iptr, d);
 592                         break;
 593
 594                 case ICMD_IINC:
 595                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 596                                       /* sx.val.i = constant                             */
 597
 598                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 599                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 600
 601                         /* Using inc and dec is not faster than add (tested with
 602                            sieve). */
 603
 604                         M_INTMOVE(s1, d);
 605                         M_IADD_IMM(iptr->sx.val.i, d);
 606                         emit_store_dst(jd, iptr, d);
 607                         break;
 608
 609                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 610
 611                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 612                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 613                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 614                         if (s2 == d)
 615                                 M_LADD(s1, d);
 616                         else {
 617                                 M_INTMOVE(s1, d);
 618                                 M_LADD(s2, d);
 619                         }
 620                         emit_store_dst(jd, iptr, d);
 621                         break;
 622
 623                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 624                                       /* sx.val.l = constant                             */
 625
 626                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 627                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 628                         M_INTMOVE(s1, d);
 629                         if (IS_IMM32(iptr->sx.val.l))
 630                                 M_LADD_IMM(iptr->sx.val.l, d);
 631                         else {
 632                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 633                                 M_LADD(REG_ITMP2, d);
 634                         }
 635                         emit_store_dst(jd, iptr, d);
 636                         break;
 637
 638                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 639
 640                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 641                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 642                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 643                         if (s2 == d) {
 644                                 M_INTMOVE(s1, REG_ITMP1);
 645                                 M_ISUB(s2, REG_ITMP1);
 646                                 M_INTMOVE(REG_ITMP1, d);
 647                         } else {
 648                                 M_INTMOVE(s1, d);
 649                                 M_ISUB(s2, d);
 650                         }
 651                         emit_store_dst(jd, iptr, d);
 652                         break;
 653
 654                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 655                                       /* sx.val.i = constant                             */
 656
 657                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 658                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 659                         M_INTMOVE(s1, d);
 660                         M_ISUB_IMM(iptr->sx.val.i, d);
 661                         emit_store_dst(jd, iptr, d);
 662                         break;
 663
 664                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 665
 666                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 667                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 668                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 669                         if (s2 == d) {
 670                                 M_INTMOVE(s1, REG_ITMP1);
 671                                 M_LSUB(s2, REG_ITMP1);
 672                                 M_INTMOVE(REG_ITMP1, d);
 673                         } else {
 674                                 M_INTMOVE(s1, d);
 675                                 M_LSUB(s2, d);
 676                         }
 677                         emit_store_dst(jd, iptr, d);
 678                         break;
 679
 680                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 681                                       /* sx.val.l = constant                             */
 682
 683                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 684                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 685                         M_INTMOVE(s1, d);
 686                         if (IS_IMM32(iptr->sx.val.l))
 687                                 M_LSUB_IMM(iptr->sx.val.l, d);
 688                         else {
 689                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 690                                 M_LSUB(REG_ITMP2, d);
 691                         }
 692                         emit_store_dst(jd, iptr, d);
 693                         break;
 694
 695                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 696
 697                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 698                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 699                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 700                         if (s2 == d)
 701                                 M_IMUL(s1, d);
 702                         else {
 703                                 M_INTMOVE(s1, d);
 704                                 M_IMUL(s2, d);
 705                         }
 706                         emit_store_dst(jd, iptr, d);
 707                         break;
 708
 709                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 710                                       /* sx.val.i = constant                             */
 711
 712                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 713                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 714                         if (iptr->sx.val.i == 2) {
 715                                 M_INTMOVE(s1, d);
 716                                 M_ISLL_IMM(1, d);
 717                         } else
 718                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 719                         emit_store_dst(jd, iptr, d);
 720                         break;
 721
 722                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 723
 724                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 725                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 726                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 727                         if (s2 == d)
 728                                 M_LMUL(s1, d);
 729                         else {
 730                                 M_INTMOVE(s1, d);
 731                                 M_LMUL(s2, d);
 732                         }
 733                         emit_store_dst(jd, iptr, d);
 734                         break;
 735
 736                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 737                                       /* sx.val.l = constant                             */
 738
 739                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 740                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 741                         if (IS_IMM32(iptr->sx.val.l))
 742                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 743                         else {
 744                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 745                                 M_INTMOVE(s1, d);
 746                                 M_LMUL(REG_ITMP2, d);
 747                         }
 748                         emit_store_dst(jd, iptr, d);
 749                         break;
 750
 751                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 752
 753                         s1 = emit_load_s1(jd, iptr, RAX);
 754                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 755                         d = codegen_reg_of_dst(jd, iptr, RAX);
 756
 757                         M_INTMOVE(s1, RAX);
 758                         M_INTMOVE(s2, REG_ITMP3);
 759                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 760
 761                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 762
 763                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 764                         M_BNE(4 + 6);
 765                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 766                         M_BEQ(1 + 3);                                          /* 6 bytes */
 767
 768                         emit_cltd(cd);                                         /* 1 byte  */
 769                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 770
 771                         M_INTMOVE(RAX, d);
 772                         emit_store_dst(jd, iptr, d);
 773                         dst = VAROP(iptr->dst);
 774                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 775                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 776                         break;
 777
 778                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 779
 780                         s1 = emit_load_s1(jd, iptr, RAX);
 781                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 782                         d = codegen_reg_of_dst(jd, iptr, RDX);
 783
 784                         M_INTMOVE(s1, RAX);
 785                         M_INTMOVE(s2, REG_ITMP3);
 786                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 787
 788                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 789
 790                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 791                         M_BNE(3 + 4 + 6);
 792                         M_CLR(RDX);                                            /* 3 bytes */
 793                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 794                         M_BEQ(1 + 3);                                          /* 6 bytes */
 795
 796                         emit_cltd(cd);                                         /* 1 byte  */
 797                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 798
 799                         M_INTMOVE(RDX, d);
 800                         emit_store_dst(jd, iptr, d);
 801                         dst = VAROP(iptr->dst);
 802                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 803                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 804                         break;
 805
 806                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 807                                       /* sx.val.i = constant                             */
 808
 809                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 810                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 811                         M_INTMOVE(s1, REG_ITMP1);
 812                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 813                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 814                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 815                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 816                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 817                         emit_store_dst(jd, iptr, d);
 818                         break;
 819
 820                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 821                                       /* sx.val.i = constant                             */
 822
 823                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 824                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 825                         M_INTMOVE(s1, REG_ITMP1);
 826                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 827                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 828                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 829                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 830                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 831                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 832                         emit_store_dst(jd, iptr, d);
 833                         break;
 834
 835
 836                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 837
 838                         s1 = emit_load_s1(jd, iptr, RAX);
 839                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 840                         d = codegen_reg_of_dst(jd, iptr, RAX);
 841
 842                         M_INTMOVE(s1, RAX);
 843                         M_INTMOVE(s2, REG_ITMP3);
 844                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 845
 846                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 847
 848                         /* check as described in jvm spec */
 849                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 850                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 851                         M_BNE(4 + 6);
 852                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 853                         M_BEQ(2 + 3);                                          /* 6 bytes */
 854
 855                         emit_cqto(cd);                                         /* 2 bytes */
 856                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 857
 858                         M_INTMOVE(RAX, d);
 859                         emit_store_dst(jd, iptr, d);
 860                         dst = VAROP(iptr->dst);
 861                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 862                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 863                         break;
 864
 865                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 866
 867                         s1 = emit_load_s1(jd, iptr, RAX);
 868                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 869                         d = codegen_reg_of_dst(jd, iptr, RDX);
 870
 871                         M_INTMOVE(s1, RAX);
 872                         M_INTMOVE(s2, REG_ITMP3);
 873                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 874
 875                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 876
 877                         /* check as described in jvm spec */
 878                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 879                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 880                         M_BNE(3 + 4 + 6);
 881                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 882                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 883                         M_BEQ(2 + 3);                                          /* 6 bytes */
 884
 885                         emit_cqto(cd);                                         /* 2 bytes */
 886                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 887
 888                         M_INTMOVE(RDX, d);
 889                         emit_store_dst(jd, iptr, d);
 890                         dst = VAROP(iptr->dst);
 891                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 892                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 893                         break;
 894
 895                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 896                                       /* sx.val.i = constant                             */
 897
 898                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 899                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 900                         M_INTMOVE(s1, REG_ITMP1);
 901                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 902                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 903                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 904                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 905                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 906                         emit_store_dst(jd, iptr, d);
 907                         break;
 908
 909                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 910                                       /* sx.val.l = constant                             */
 911
 912                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 913                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 914                         M_INTMOVE(s1, REG_ITMP1);
 915                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 916                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 917                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 918                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 919                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 920                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 921                         emit_store_dst(jd, iptr, d);
 922                         break;
 923
 924                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 925
 926                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 927                         emit_ishift(jd, SHIFT_SHL, iptr);
 928                         break;
 929
 930                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 931                                       /* sx.val.i = constant                             */
 932
 933                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 934                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 935                         M_INTMOVE(s1, d);
 936                         M_ISLL_IMM(iptr->sx.val.i, d);
 937                         emit_store_dst(jd, iptr, d);
 938                         break;
 939
 940                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 941
 942                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 943                         emit_ishift(jd, SHIFT_SAR, iptr);
 944                         break;
 945
 946                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 947                                       /* sx.val.i = constant                             */
 948
 949                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 950                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 951                         M_INTMOVE(s1, d);
 952                         M_ISRA_IMM(iptr->sx.val.i, d);
 953                         emit_store_dst(jd, iptr, d);
 954                         break;
 955
 956                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 957
 958                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 959                         emit_ishift(jd, SHIFT_SHR, iptr);
 960                         break;
 961
 962                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 963                                       /* sx.val.i = constant                             */
 964
 965                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 966                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 967                         M_INTMOVE(s1, d);
 968                         M_ISRL_IMM(iptr->sx.val.i, d);
 969                         emit_store_dst(jd, iptr, d);
 970                         break;
 971
 972                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 973
 974                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 975                         emit_lshift(jd, SHIFT_SHL, iptr);
 976                         break;
 977
 978         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 979                                           /* sx.val.i = constant                             */
 980
 981                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 982                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 983                         M_INTMOVE(s1, d);
 984                         M_LSLL_IMM(iptr->sx.val.i, d);
 985                         emit_store_dst(jd, iptr, d);
 986                         break;
 987
 988                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 989
 990                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 991                         emit_lshift(jd, SHIFT_SAR, iptr);
 992                         break;
 993
 994                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 995                                       /* sx.val.i = constant                             */
 996
 997                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 998                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 999                         M_INTMOVE(s1, d);
1000                         M_LSRA_IMM(iptr->sx.val.i, d);
1001                         emit_store_dst(jd, iptr, d);
1002                         break;
1003
1004                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1005
1006                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1007                         emit_lshift(jd, SHIFT_SHR, iptr);
1008                         break;
1009
1010                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1011                                       /* sx.val.l = constant                             */
1012
1013                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1014                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1015                         M_INTMOVE(s1, d);
1016                         M_LSRL_IMM(iptr->sx.val.i, d);
1017                         emit_store_dst(jd, iptr, d);
1018                         break;
1019
1020                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1021
1022                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1023                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1024                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1025                         if (s2 == d)
1026                                 M_IAND(s1, d);
1027                         else {
1028                                 M_INTMOVE(s1, d);
1029                                 M_IAND(s2, d);
1030                         }
1031                         emit_store_dst(jd, iptr, d);
1032                         break;
1033
1034                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1035                                       /* sx.val.i = constant                             */
1036
1037                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1038                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1039                         M_INTMOVE(s1, d);
1040                         M_IAND_IMM(iptr->sx.val.i, d);
1041                         emit_store_dst(jd, iptr, d);
1042                         break;
1043
1044                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1045
1046                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1047                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1048                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1049                         if (s2 == d)
1050                                 M_LAND(s1, d);
1051                         else {
1052                                 M_INTMOVE(s1, d);
1053                                 M_LAND(s2, d);
1054                         }
1055                         emit_store_dst(jd, iptr, d);
1056                         break;
1057
1058                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1059                                       /* sx.val.l = constant                             */
1060
1061                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1062                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1063                         M_INTMOVE(s1, d);
1064                         if (IS_IMM32(iptr->sx.val.l))
1065                                 M_LAND_IMM(iptr->sx.val.l, d);
1066                         else {
1067                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1068                                 M_LAND(REG_ITMP2, d);
1069                         }
1070                         emit_store_dst(jd, iptr, d);
1071                         break;
1072
1073                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1074
1075                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1076                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1077                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1078                         if (s2 == d)
1079                                 M_IOR(s1, d);
1080                         else {
1081                                 M_INTMOVE(s1, d);
1082                                 M_IOR(s2, d);
1083                         }
1084                         emit_store_dst(jd, iptr, d);
1085                         break;
1086
1087                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1088                                       /* sx.val.i = constant                             */
1089
1090                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1091                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1092                         M_INTMOVE(s1, d);
1093                         M_IOR_IMM(iptr->sx.val.i, d);
1094                         emit_store_dst(jd, iptr, d);
1095                         break;
1096
1097                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1098
1099                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1100                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1101                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1102                         if (s2 == d)
1103                                 M_LOR(s1, d);
1104                         else {
1105                                 M_INTMOVE(s1, d);
1106                                 M_LOR(s2, d);
1107                         }
1108                         emit_store_dst(jd, iptr, d);
1109                         break;
1110
1111                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1112                                       /* sx.val.l = constant                             */
1113
1114                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1115                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1116                         M_INTMOVE(s1, d);
1117                         if (IS_IMM32(iptr->sx.val.l))
1118                                 M_LOR_IMM(iptr->sx.val.l, d);
1119                         else {
1120                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1121                                 M_LOR(REG_ITMP2, d);
1122                         }
1123                         emit_store_dst(jd, iptr, d);
1124                         break;
1125
1126                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1127
1128                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1129                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1130                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1131                         if (s2 == d)
1132                                 M_IXOR(s1, d);
1133                         else {
1134                                 M_INTMOVE(s1, d);
1135                                 M_IXOR(s2, d);
1136                         }
1137                         emit_store_dst(jd, iptr, d);
1138                         break;
1139
1140                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1141                                       /* sx.val.i = constant                             */
1142
1143                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1144                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1145                         M_INTMOVE(s1, d);
1146                         M_IXOR_IMM(iptr->sx.val.i, d);
1147                         emit_store_dst(jd, iptr, d);
1148                         break;
1149
1150                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1151
1152                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1153                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1154                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1155                         if (s2 == d)
1156                                 M_LXOR(s1, d);
1157                         else {
1158                                 M_INTMOVE(s1, d);
1159                                 M_LXOR(s2, d);
1160                         }
1161                         emit_store_dst(jd, iptr, d);
1162                         break;
1163
1164                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1165                                       /* sx.val.l = constant                             */
1166
1167                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1168                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1169                         M_INTMOVE(s1, d);
1170                         if (IS_IMM32(iptr->sx.val.l))
1171                                 M_LXOR_IMM(iptr->sx.val.l, d);
1172                         else {
1173                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1174                                 M_LXOR(REG_ITMP2, d);
1175                         }
1176                         emit_store_dst(jd, iptr, d);
1177                         break;
1178
1179
1180                 /* floating operations ************************************************/
1181
1182                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1183
1184                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1185                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1186                         disp = dseg_add_s4(cd, 0x80000000);
1187                         M_FLTMOVE(s1, d);
1188                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1189                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1190                         emit_store_dst(jd, iptr, d);
1191                         break;
1192
1193                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1194
1195                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1196                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1197                         disp = dseg_add_s8(cd, 0x8000000000000000);
1198                         M_FLTMOVE(s1, d);
1199                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1200                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1201                         emit_store_dst(jd, iptr, d);
1202                         break;
1203
1204                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1205
1206                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1207                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1208                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1209                         if (s2 == d)
1210                                 M_FADD(s1, d);
1211                         else {
1212                                 M_FLTMOVE(s1, d);
1213                                 M_FADD(s2, d);
1214                         }
1215                         emit_store_dst(jd, iptr, d);
1216                         break;
1217
1218                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1219
1220                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1221                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1222                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1223                         if (s2 == d)
1224                                 M_DADD(s1, d);
1225                         else {
1226                                 M_FLTMOVE(s1, d);
1227                                 M_DADD(s2, d);
1228                         }
1229                         emit_store_dst(jd, iptr, d);
1230                         break;
1231
1232                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1233
1234                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1235                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1236                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1237                         if (s2 == d) {
1238                                 M_FLTMOVE(s2, REG_FTMP2);
1239                                 s2 = REG_FTMP2;
1240                         }
1241                         M_FLTMOVE(s1, d);
1242                         M_FSUB(s2, d);
1243                         emit_store_dst(jd, iptr, d);
1244                         break;
1245
1246                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1247
1248                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1249                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1250                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1251                         if (s2 == d) {
1252                                 M_FLTMOVE(s2, REG_FTMP2);
1253                                 s2 = REG_FTMP2;
1254                         }
1255                         M_FLTMOVE(s1, d);
1256                         M_DSUB(s2, d);
1257                         emit_store_dst(jd, iptr, d);
1258                         break;
1259
1260                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1261
1262                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1263                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1264                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1265                         if (s2 == d)
1266                                 M_FMUL(s1, d);
1267                         else {
1268                                 M_FLTMOVE(s1, d);
1269                                 M_FMUL(s2, d);
1270                         }
1271                         emit_store_dst(jd, iptr, d);
1272                         break;
1273
1274                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1275
1276                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1277                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1278                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1279                         if (s2 == d)
1280                                 M_DMUL(s1, d);
1281                         else {
1282                                 M_FLTMOVE(s1, d);
1283                                 M_DMUL(s2, d);
1284                         }
1285                         emit_store_dst(jd, iptr, d);
1286                         break;
1287
1288                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1289
1290                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1291                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1292                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1293                         if (s2 == d) {
1294                                 M_FLTMOVE(s2, REG_FTMP2);
1295                                 s2 = REG_FTMP2;
1296                         }
1297                         M_FLTMOVE(s1, d);
1298                         M_FDIV(s2, d);
1299                         emit_store_dst(jd, iptr, d);
1300                         break;
1301
1302                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1303
1304                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1305                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1306                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1307                         if (s2 == d) {
1308                                 M_FLTMOVE(s2, REG_FTMP2);
1309                                 s2 = REG_FTMP2;
1310                         }
1311                         M_FLTMOVE(s1, d);
1312                         M_DDIV(s2, d);
1313                         emit_store_dst(jd, iptr, d);
1314                         break;
1315
1316                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1317
1318                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1319                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1320                         M_CVTIF(s1, d);
1321                         emit_store_dst(jd, iptr, d);
1322                         break;
1323
1324                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1325
1326                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1327                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1328                         M_CVTID(s1, d);
1329                         emit_store_dst(jd, iptr, d);
1330                         break;
1331
1332                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1333
1334                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1335                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1336                         M_CVTLF(s1, d);
1337                         emit_store_dst(jd, iptr, d);
1338                         break;
1339
1340                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1341
1342                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1343                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1344                         M_CVTLD(s1, d);
1345                         emit_store_dst(jd, iptr, d);
1346                         break;
1347
1348                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1349
1350                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1351                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1352                         M_CVTFI(s1, d);
1353                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1354                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1355                                 ((REG_RESULT == d) ? 0 : 3);
1356                         M_BNE(disp);
1357                         M_FLTMOVE(s1, REG_FTMP1);
1358                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1359                         M_CALL(REG_ITMP2);
1360                         M_INTMOVE(REG_RESULT, d);
1361                         emit_store_dst(jd, iptr, d);
1362                         break;
1363
1364                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1365
1366                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1367                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1368                         M_CVTDI(s1, d);
1369                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1370                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1371                                 ((REG_RESULT == d) ? 0 : 3);
1372                         M_BNE(disp);
1373                         M_FLTMOVE(s1, REG_FTMP1);
1374                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1375                         M_CALL(REG_ITMP2);
1376                         M_INTMOVE(REG_RESULT, d);
1377                         emit_store_dst(jd, iptr, d);
1378                         break;
1379
1380                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1381
1382                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1383                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1384                         M_CVTFL(s1, d);
1385                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1386                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1387                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1388                                 ((REG_RESULT == d) ? 0 : 3);
1389                         M_BNE(disp);
1390                         M_FLTMOVE(s1, REG_FTMP1);
1391                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1392                         M_CALL(REG_ITMP2);
1393                         M_INTMOVE(REG_RESULT, d);
1394                         emit_store_dst(jd, iptr, d);
1395                         break;
1396
1397                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1398
1399                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1400                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1401                         M_CVTDL(s1, d);
1402                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1403                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1404                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1405                                 ((REG_RESULT == d) ? 0 : 3);
1406                         M_BNE(disp);
1407                         M_FLTMOVE(s1, REG_FTMP1);
1408                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1409                         M_CALL(REG_ITMP2);
1410                         M_INTMOVE(REG_RESULT, d);
1411                         emit_store_dst(jd, iptr, d);
1412                         break;
1413
1414                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1415
1416                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1417                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1418                         M_CVTFD(s1, d);
1419                         emit_store_dst(jd, iptr, d);
1420                         break;
1421
1422                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1423
1424                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1425                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1426                         M_CVTDF(s1, d);
1427                         emit_store_dst(jd, iptr, d);
1428                         break;
1429
1430                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1431                                           /* == => 0, < => 1, > => -1 */
1432
1433                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1434                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1435                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1436                         M_CLR(d);
1437                         M_MOV_IMM(1, REG_ITMP1);
1438                         M_MOV_IMM(-1, REG_ITMP2);
1439                         emit_ucomiss_reg_reg(cd, s1, s2);
1440                         M_CMOVULT(REG_ITMP1, d);
1441                         M_CMOVUGT(REG_ITMP2, d);
1442                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1443                         emit_store_dst(jd, iptr, d);
1444                         break;
1445
1446                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1447                                           /* == => 0, < => 1, > => -1 */
1448
1449                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1450                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1451                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1452                         M_CLR(d);
1453                         M_MOV_IMM(1, REG_ITMP1);
1454                         M_MOV_IMM(-1, REG_ITMP2);
1455                         emit_ucomiss_reg_reg(cd, s1, s2);
1456                         M_CMOVULT(REG_ITMP1, d);
1457                         M_CMOVUGT(REG_ITMP2, d);
1458                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1459                         emit_store_dst(jd, iptr, d);
1460                         break;
1461
1462                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1463                                           /* == => 0, < => 1, > => -1 */
1464
1465                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1466                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1467                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1468                         M_CLR(d);
1469                         M_MOV_IMM(1, REG_ITMP1);
1470                         M_MOV_IMM(-1, REG_ITMP2);
1471                         emit_ucomisd_reg_reg(cd, s1, s2);
1472                         M_CMOVULT(REG_ITMP1, d);
1473                         M_CMOVUGT(REG_ITMP2, d);
1474                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1475                         emit_store_dst(jd, iptr, d);
1476                         break;
1477
1478                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1479                                           /* == => 0, < => 1, > => -1 */
1480
1481                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1482                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1483                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1484                         M_CLR(d);
1485                         M_MOV_IMM(1, REG_ITMP1);
1486                         M_MOV_IMM(-1, REG_ITMP2);
1487                         emit_ucomisd_reg_reg(cd, s1, s2);
1488                         M_CMOVULT(REG_ITMP1, d);
1489                         M_CMOVUGT(REG_ITMP2, d);
1490                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1491                         emit_store_dst(jd, iptr, d);
1492                         break;
1493
1494
1495                 /* memory operations **************************************************/
1496
1497                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1498
1499                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1500                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1501                         /* implicit null-pointer check */
1502                         M_ILD(d, s1, OFFSET(java_array_t, size));
1503                         emit_store_dst(jd, iptr, d);
1504                         break;
1505
1506                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1507
1508                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1509                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1510                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1511                         /* implicit null-pointer check */
1512                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1513                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1514                         emit_store_dst(jd, iptr, d);
1515                         break;
1516
1517                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1518
1519                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1520                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1521                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1522                         /* implicit null-pointer check */
1523                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1524                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1525                         emit_store_dst(jd, iptr, d);
1526                         break;
1527
1528                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1529
1530                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1531                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1532                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1533                         /* implicit null-pointer check */
1534                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1535                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1536                         emit_store_dst(jd, iptr, d);
1537                         break;
1538
1539                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1540
1541                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1542                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1543                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1544                         /* implicit null-pointer check */
1545                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1546                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1547                         emit_store_dst(jd, iptr, d);
1548                         break;
1549
1550                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1551
1552                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1553                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1554                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1555                         /* implicit null-pointer check */
1556                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1557                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1558                         emit_store_dst(jd, iptr, d);
1559                         break;
1560
1561                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1562
1563                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1564                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1565                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1566                         /* implicit null-pointer check */
1567                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1568                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1569                         emit_store_dst(jd, iptr, d);
1570                         break;
1571
1572                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1573
1574                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1575                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1576                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1577                         /* implicit null-pointer check */
1578                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1579                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1580                         emit_store_dst(jd, iptr, d);
1581                         break;
1582
1583                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1584
1585                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1586                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1587                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1588                         /* implicit null-pointer check */
1589                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1590                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1591                         emit_store_dst(jd, iptr, d);
1592                         break;
1593
1594
1595                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1596
1597                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1598                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1599                         /* implicit null-pointer check */
1600                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1601                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1602                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1603                         break;
1604
1605                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1606
1607                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1608                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1609                         /* implicit null-pointer check */
1610                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1611                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1612                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1613                         break;
1614
1615                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1616
1617                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1618                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1619                         /* implicit null-pointer check */
1620                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1621                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1622                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1623                         break;
1624
1625                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1626
1627                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1628                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1629                         /* implicit null-pointer check */
1630                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1631                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1632                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1633                         break;
1634
1635                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1636
1637                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1638                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1639                         /* implicit null-pointer check */
1640                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1641                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1642                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1643                         break;
1644
1645                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1646
1647                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1648                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1649                         /* implicit null-pointer check */
1650                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1651                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1652                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1653                         break;
1654
1655                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1656
1657                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1658                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1659                         /* implicit null-pointer check */
1660                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1661                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1662                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1663                         break;
1664
1665                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1666
1667                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1668                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1669                         /* implicit null-pointer check */
1670                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1671                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1672
1673                         M_MOV(s1, REG_A0);
1674                         M_MOV(s3, REG_A1);
1675                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
1676                         M_CALL(REG_ITMP1);
1677                         emit_exception_check(cd, iptr);
1678
1679                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1680                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1681                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1682                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1683                         break;
1684
1685
1686                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1687
1688                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1689                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1690                         /* implicit null-pointer check */
1691                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1692                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1693                         break;
1694
1695                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1696
1697                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1698                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1699                         /* implicit null-pointer check */
1700                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1701                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1702                         break;
1703
1704                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1705
1706                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1707                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1708                         /* implicit null-pointer check */
1709                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1710                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1711                         break;
1712
1713                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1714
1715                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1716                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1717                         /* implicit null-pointer check */
1718                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1719                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1720                         break;
1721
1722                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1723
1724                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1725                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1726                         /* implicit null-pointer check */
1727                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1728
1729                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1730                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1731                         }
1732                         else {
1733                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1734                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1735                         }
1736                         break;
1737
1738                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1739
1740                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1741                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1742                         /* implicit null-pointer check */
1743                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1744                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1745                         break;
1746
1747
1748                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1749
1750                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1751                                 uf        = iptr->sx.s23.s3.uf;
1752                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1753                                 disp      = dseg_add_unique_address(cd, NULL);
1754                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1755
1756                                 /* must be calculated before codegen_add_patch_ref */
1757
1758                                 if (opt_shownops)
1759                                         disp -= PATCHER_CALL_SIZE;
1760
1761 /*                              PROFILE_CYCLE_STOP; */
1762
1763                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1764
1765 /*                              PROFILE_CYCLE_START; */
1766                         }
1767                         else {
1768                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1769                                 fieldtype = fi->type;
1770                                 disp      = dseg_add_address(cd, fi->value);
1771                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1772
1773                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1774                                         PROFILE_CYCLE_STOP;
1775
1776                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1777
1778                                         PROFILE_CYCLE_START;
1779
1780                                         if (opt_shownops)
1781                                                 disp -= PATCHER_CALL_SIZE;
1782                                 }
1783                         }
1784
1785                         /* This approach is much faster than moving the field
1786                            address inline into a register. */
1787
1788                         M_ALD(REG_ITMP1, RIP, disp);
1789
1790                         switch (fieldtype) {
1791                         case TYPE_INT:
1792                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1793                                 M_ILD(d, REG_ITMP1, 0);
1794                                 break;
1795                         case TYPE_LNG:
1796                         case TYPE_ADR:
1797                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1798                                 M_LLD(d, REG_ITMP1, 0);
1799                                 break;
1800                         case TYPE_FLT:
1801                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1802                                 M_FLD(d, REG_ITMP1, 0);
1803                                 break;
1804                         case TYPE_DBL:
1805                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1806                                 M_DLD(d, REG_ITMP1, 0);
1807                                 break;
1808                         }
1809                         emit_store_dst(jd, iptr, d);
1810                         break;
1811
1812                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1813
1814                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1815                                 uf        = iptr->sx.s23.s3.uf;
1816                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1817                                 disp      = dseg_add_unique_address(cd, NULL);
1818                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1819
1820                                 /* must be calculated before codegen_add_patch_ref */
1821
1822                                 if (opt_shownops)
1823                                         disp -= PATCHER_CALL_SIZE;
1824
1825 /*                              PROFILE_CYCLE_STOP; */
1826
1827                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1828
1829 /*                              PROFILE_CYCLE_START; */
1830                         }
1831                         else {
1832                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1833                                 fieldtype = fi->type;
1834                                 disp      = dseg_add_address(cd, fi->value);
1835                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1836
1837                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1838                                         PROFILE_CYCLE_STOP;
1839
1840                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1841
1842                                         PROFILE_CYCLE_START;
1843
1844                                         if (opt_shownops)
1845                                                 disp -= PATCHER_CALL_SIZE;
1846                                 }
1847                         }
1848
1849                         /* This approach is much faster than moving the field
1850                            address inline into a register. */
1851
1852                         M_ALD(REG_ITMP1, RIP, disp);
1853
1854                         switch (fieldtype) {
1855                         case TYPE_INT:
1856                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1857                                 M_IST(s1, REG_ITMP1, 0);
1858                                 break;
1859                         case TYPE_LNG:
1860                         case TYPE_ADR:
1861                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1862                                 M_LST(s1, REG_ITMP1, 0);
1863                                 break;
1864                         case TYPE_FLT:
1865                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1866                                 M_FST(s1, REG_ITMP1, 0);
1867                                 break;
1868                         case TYPE_DBL:
1869                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1870                                 M_DST(s1, REG_ITMP1, 0);
1871                                 break;
1872                         }
1873                         break;
1874
1875                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1876                                           /* val = value (in current instruction)     */
1877                                           /* following NOP)                           */
1878
1879                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1880                                 uf        = iptr->sx.s23.s3.uf;
1881                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1882                                 disp      = dseg_add_unique_address(cd, NULL);
1883                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1884
1885                                 /* must be calculated before codegen_add_patch_ref */
1886
1887                                 if (opt_shownops)
1888                                         disp -= PATCHER_CALL_SIZE;
1889
1890 /*                              PROFILE_CYCLE_STOP; */
1891
1892                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1893
1894 /*                              PROFILE_CYCLE_START; */
1895                         }
1896                         else {
1897                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1898                                 fieldtype = fi->type;
1899                                 disp      = dseg_add_address(cd, fi->value);
1900                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1901
1902                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1903                                         PROFILE_CYCLE_STOP;
1904
1905                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1906
1907                                         PROFILE_CYCLE_START;
1908
1909                                         if (opt_shownops)
1910                                                 disp -= PATCHER_CALL_SIZE;
1911                                 }
1912                         }
1913
1914                         /* This approach is much faster than moving the field
1915                            address inline into a register. */
1916
1917                         M_ALD(REG_ITMP1, RIP, disp);
1918
1919                         switch (fieldtype) {
1920                         case TYPE_INT:
1921                         case TYPE_FLT:
1922                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1923                                 break;
1924                         case TYPE_LNG:
1925                         case TYPE_ADR:
1926                         case TYPE_DBL:
1927                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1928                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1929                                 else {
1930                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1931                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1932                                 }
1933                                 break;
1934                         }
1935                         break;
1936
1937                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1938
1939                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1940
1941                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1942                                 uf        = iptr->sx.s23.s3.uf;
1943                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1944                                 disp      = 0;
1945
1946 /*                              PROFILE_CYCLE_STOP; */
1947
1948                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1949
1950 /*                              PROFILE_CYCLE_START; */
1951                         }
1952                         else {
1953                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1954                                 fieldtype = fi->type;
1955                                 disp      = fi->offset;
1956                         }
1957
1958                         /* implicit null-pointer check */
1959                         switch (fieldtype) {
1960                         case TYPE_INT:
1961                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1962                                 M_ILD32(d, s1, disp);
1963                                 break;
1964                         case TYPE_LNG:
1965                         case TYPE_ADR:
1966                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1967                                 M_LLD32(d, s1, disp);
1968                                 break;
1969                         case TYPE_FLT:
1970                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1971                                 M_FLD32(d, s1, disp);
1972                                 break;
1973                         case TYPE_DBL:
1974                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1975                                 M_DLD32(d, s1, disp);
1976                                 break;
1977                         }
1978                         emit_store_dst(jd, iptr, d);
1979                         break;
1980
1981                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1982
1983                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1984                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1985
1986                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1987                                 uf        = iptr->sx.s23.s3.uf;
1988                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1989                                 disp      = 0;
1990
1991 /*                              PROFILE_CYCLE_STOP; */
1992
1993                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1994
1995 /*                              PROFILE_CYCLE_START; */
1996                         }
1997                         else {
1998                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1999                                 fieldtype = fi->type;
2000                                 disp      = fi->offset;
2001                         }
2002
2003                         /* implicit null-pointer check */
2004                         switch (fieldtype) {
2005                         case TYPE_INT:
2006                                 M_IST32(s2, s1, disp);
2007                                 break;
2008                         case TYPE_LNG:
2009                         case TYPE_ADR:
2010                                 M_LST32(s2, s1, disp);
2011                                 break;
2012                         case TYPE_FLT:
2013                                 M_FST32(s2, s1, disp);
2014                                 break;
2015                         case TYPE_DBL:
2016                                 M_DST32(s2, s1, disp);
2017                                 break;
2018                         }
2019                         break;
2020
2021                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2022                                           /* val = value (in current instruction)     */
2023                                           /* following NOP)                           */
2024
2025                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2026
2027                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2028                                 uf        = iptr->sx.s23.s3.uf;
2029                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2030                                 disp      = 0;
2031
2032 /*                              PROFILE_CYCLE_STOP; */
2033
2034                                 codegen_add_patch_ref(cd, PATCHER_putfieldconst, uf, 0);
2035
2036 /*                              PROFILE_CYCLE_START; */
2037                         }
2038                         else {
2039                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2040                                 fieldtype = fi->type;
2041                                 disp      = fi->offset;
2042                         }
2043
2044                         /* implicit null-pointer check */
2045                         switch (fieldtype) {
2046                         case TYPE_INT:
2047                         case TYPE_FLT:
2048                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2049                                 break;
2050                         case TYPE_LNG:
2051                         case TYPE_ADR:
2052                         case TYPE_DBL:
2053                                 /* XXX why no check for IS_IMM32? */
2054                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2055                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2056                                 break;
2057                         }
2058                         break;
2059
2060
2061                 /* branch operations **************************************************/
2062
2063                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2064
2065                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2066                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2067
2068                         PROFILE_CYCLE_STOP;
2069
2070 #ifdef ENABLE_VERIFIER
2071                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2072                                 uc = iptr->sx.s23.s2.uc;
2073
2074                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2075                         }
2076 #endif /* ENABLE_VERIFIER */
2077
2078                         M_CALL_IMM(0);                            /* passing exception pc */
2079                         M_POP(REG_ITMP2_XPC);
2080
2081                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2082                         M_JMP(REG_ITMP3);
2083                         break;
2084
2085                 case ICMD_GOTO:         /* ... ==> ...                                */
2086                 case ICMD_RET:
2087
2088                         emit_br(cd, iptr->dst.block);
2089                         ALIGNCODENOP;
2090                         break;
2091
2092                 case ICMD_JSR:          /* ... ==> ...                                */
2093
2094                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2095                         ALIGNCODENOP;
2096                         break;
2097
2098                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2099                 case ICMD_IFNONNULL:
2100
2101                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2102                         M_TEST(s1);
2103                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2104                         break;
2105
2106                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2107                 case ICMD_IFLT:
2108                 case ICMD_IFLE:
2109                 case ICMD_IFNE:
2110                 case ICMD_IFGT:
2111                 case ICMD_IFGE:
2112
2113                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2114                         M_ICMP_IMM(iptr->sx.val.i, s1);
2115                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2116                         break;
2117
2118                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2119                 case ICMD_IF_LNE:
2120                 case ICMD_IF_LLT:
2121                 case ICMD_IF_LGE:
2122                 case ICMD_IF_LGT:
2123                 case ICMD_IF_LLE:
2124
2125                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2126                         if (IS_IMM32(iptr->sx.val.l))
2127                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2128                         else {
2129                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2130                                 M_LCMP(REG_ITMP2, s1);
2131                         }
2132                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2133                         break;
2134
2135                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2136                 case ICMD_IF_ICMPNE:
2137                 case ICMD_IF_ICMPLT:
2138                 case ICMD_IF_ICMPGE:
2139                 case ICMD_IF_ICMPGT:
2140                 case ICMD_IF_ICMPLE:
2141
2142                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2143                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2144                         M_ICMP(s2, s1);
2145                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2146                         break;
2147
2148                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2149                 case ICMD_IF_ACMPNE:
2150
2151                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2152                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2153                         M_LCMP(s2, s1);
2154                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2155                         break;
2156
2157                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2158                 case ICMD_IF_LCMPNE:
2159                 case ICMD_IF_LCMPLT:
2160                 case ICMD_IF_LCMPGE:
2161                 case ICMD_IF_LCMPGT:
2162                 case ICMD_IF_LCMPLE:
2163
2164                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2165                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2166                         M_LCMP(s2, s1);
2167                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2168                         break;
2169
2170                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2171                 case ICMD_LRETURN:
2172
2173                         REPLACEMENT_POINT_RETURN(cd, iptr);
2174                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2175                         M_INTMOVE(s1, REG_RESULT);
2176                         goto nowperformreturn;
2177
2178                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2179
2180                         REPLACEMENT_POINT_RETURN(cd, iptr);
2181                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2182                         M_INTMOVE(s1, REG_RESULT);
2183
2184 #ifdef ENABLE_VERIFIER
2185                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2186                                 uc = iptr->sx.s23.s2.uc;
2187
2188                                 PROFILE_CYCLE_STOP;
2189
2190                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2191
2192                                 PROFILE_CYCLE_START;
2193                         }
2194 #endif /* ENABLE_VERIFIER */
2195                         goto nowperformreturn;
2196
2197                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2198                 case ICMD_DRETURN:
2199
2200                         REPLACEMENT_POINT_RETURN(cd, iptr);
2201                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2202                         M_FLTMOVE(s1, REG_FRESULT);
2203                         goto nowperformreturn;
2204
2205                 case ICMD_RETURN:      /* ...  ==> ...                                */
2206
2207                         REPLACEMENT_POINT_RETURN(cd, iptr);
2208
2209 nowperformreturn:
2210                         {
2211                         s4 i, p;
2212
2213                         p = cd->stackframesize;
2214
2215 #if !defined(NDEBUG)
2216                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2217                                 emit_verbosecall_exit(jd);
2218 #endif /* !defined(NDEBUG) */
2219
2220 #if defined(ENABLE_THREADS)
2221                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2222                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2223
2224                                 /* we need to save the proper return value */
2225                                 switch (iptr->opc) {
2226                                 case ICMD_IRETURN:
2227                                 case ICMD_ARETURN:
2228                                 case ICMD_LRETURN:
2229                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2230                                         break;
2231                                 case ICMD_FRETURN:
2232                                 case ICMD_DRETURN:
2233                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2234                                         break;
2235                                 }
2236
2237                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2238                                 M_CALL(REG_ITMP1);
2239
2240                                 /* and now restore the proper return value */
2241                                 switch (iptr->opc) {
2242                                 case ICMD_IRETURN:
2243                                 case ICMD_ARETURN:
2244                                 case ICMD_LRETURN:
2245                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2246                                         break;
2247                                 case ICMD_FRETURN:
2248                                 case ICMD_DRETURN:
2249                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2250                                         break;
2251                                 }
2252                         }
2253 #endif
2254
2255                         /* restore saved registers */
2256
2257                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2258                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2259                         }
2260                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2261                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2262                         }
2263
2264                         /* deallocate stack */
2265
2266                         if (cd->stackframesize)
2267                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2268
2269                         /* generate method profiling code */
2270
2271                         PROFILE_CYCLE_STOP;
2272
2273                         M_RET;
2274                         }
2275                         break;
2276
2277
2278                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2279                         {
2280                                 s4 i, l;
2281                                 branch_target_t *table;
2282
2283                                 table = iptr->dst.table;
2284
2285                                 l = iptr->sx.s23.s2.tablelow;
2286                                 i = iptr->sx.s23.s3.tablehigh;
2287
2288                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2289                                 M_INTMOVE(s1, REG_ITMP1);
2290
2291                                 if (l != 0)
2292                                         M_ISUB_IMM(l, REG_ITMP1);
2293
2294                                 /* number of targets */
2295                                 i = i - l + 1;
2296
2297                 /* range check */
2298
2299                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2300                                 emit_bugt(cd, table[0].block);
2301
2302                                 /* build jump table top down and use address of lowest entry */
2303
2304                                 table += i;
2305
2306                                 while (--i >= 0) {
2307                                         dseg_add_target(cd, table->block);
2308                                         --table;
2309                                 }
2310
2311                                 /* length of dataseg after last dseg_add_target is used
2312                                    by load */
2313
2314                                 M_MOV_IMM(0, REG_ITMP2);
2315                                 dseg_adddata(cd);
2316                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2317                                 M_JMP(REG_ITMP1);
2318                         }
2319                         break;
2320
2321
2322                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2323                         {
2324                                 s4 i;
2325                                 lookup_target_t *lookup;
2326
2327                                 lookup = iptr->dst.lookup;
2328
2329                                 i = iptr->sx.s23.s2.lookupcount;
2330
2331                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2332                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2333
2334                                 while (--i >= 0) {
2335                                         M_ICMP_IMM(lookup->value, s1);
2336                                         emit_beq(cd, lookup->target.block);
2337                                         lookup++;
2338                                 }
2339
2340                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2341                                 ALIGNCODENOP;
2342                         }
2343                         break;
2344
2345
2346                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2347
2348                         REPLACEMENT_POINT_FORGC_BUILTIN(cd, iptr);
2349
2350                         bte = iptr->sx.s23.s3.bte;
2351                         md  = bte->md;
2352                         goto gen_method;
2353
2354                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2355
2356                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2357                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2358                 case ICMD_INVOKEINTERFACE:
2359
2360                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2361
2362                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2363                                 lm = NULL;
2364                                 um = iptr->sx.s23.s3.um;
2365                                 md = um->methodref->parseddesc.md;
2366                         }
2367                         else {
2368                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2369                                 um = NULL;
2370                                 md = lm->parseddesc;
2371                         }
2372
2373 gen_method:
2374                         s3 = md->paramcount;
2375
2376                         MCODECHECK((20 * s3) + 128);
2377
2378                         /* copy arguments to registers or stack location */
2379
2380                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2381                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2382                                 d   = md->params[s3].regoff;
2383
2384                                 /* already preallocated (ARGVAR)? */
2385
2386                                 if (var->flags & PREALLOC)
2387                                         continue;
2388
2389                                 if (IS_INT_LNG_TYPE(var->type)) {
2390                                         if (!md->params[s3].inmemory) {
2391                                                 s1 = emit_load(jd, iptr, var, d);
2392                                                 M_INTMOVE(s1, d);
2393                                         }
2394                                         else {
2395                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2396                                                 M_LST(s1, REG_SP, d);
2397                                         }
2398                                 }
2399                                 else {
2400                                         if (!md->params[s3].inmemory) {
2401                                                 s1 = emit_load(jd, iptr, var, d);
2402                                                 M_FLTMOVE(s1, d);
2403                                         }
2404                                         else {
2405                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2406
2407                                                 if (IS_2_WORD_TYPE(var->type))
2408                                                         M_DST(s1, REG_SP, d);
2409                                                 else
2410                                                         M_FST(s1, REG_SP, d);
2411                                         }
2412                                 }
2413                         }
2414
2415                         /* generate method profiling code */
2416
2417                         PROFILE_CYCLE_STOP;
2418
2419                         switch (iptr->opc) {
2420                         case ICMD_BUILTIN:
2421                                 if (bte->stub == NULL) {
2422                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2423                                 } else {
2424                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2425                                 }
2426                                 M_CALL(REG_ITMP1);
2427
2428                                 emit_exception_check(cd, iptr);
2429                                 break;
2430
2431                         case ICMD_INVOKESPECIAL:
2432                                 emit_nullpointer_check(cd, iptr, REG_A0);
2433                                 /* fall through */
2434
2435                         case ICMD_INVOKESTATIC:
2436                                 if (lm == NULL) {
2437                                         disp = dseg_add_unique_address(cd, NULL);
2438                                         disp = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
2439
2440                                         /* must be calculated before codegen_add_patch_ref */
2441
2442                                         if (opt_shownops)
2443                                                 disp -= PATCHER_CALL_SIZE;
2444
2445                                         codegen_add_patch_ref(cd, PATCHER_invokestatic_special,
2446                                                                                   um, disp);
2447
2448 /*                                      a = 0; */
2449                                 }
2450                                 else {
2451                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2452                                         disp = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
2453
2454 /*                                      a = (ptrint) lm->stubroutine; */
2455                                 }
2456
2457 /*                              M_MOV_IMM(a, REG_ITMP2); */
2458                                 M_ALD(REG_ITMP2, RIP, disp);
2459                                 M_CALL(REG_ITMP2);
2460                                 break;
2461
2462                         case ICMD_INVOKEVIRTUAL:
2463                                 if (lm == NULL) {
2464                                         codegen_add_patch_ref(cd, PATCHER_invokevirtual, um, 0);
2465
2466                                         s1 = 0;
2467                                 }
2468                                 else {
2469                                         s1 = OFFSET(vftbl_t, table[0]) +
2470                                                 sizeof(methodptr) * lm->vftblindex;
2471                                 }
2472
2473                                 /* implicit null-pointer check */
2474                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2475                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2476                                 M_CALL(REG_ITMP3);
2477                                 break;
2478
2479                         case ICMD_INVOKEINTERFACE:
2480                                 if (lm == NULL) {
2481                                         codegen_add_patch_ref(cd, PATCHER_invokeinterface, um, 0);
2482
2483                                         s1 = 0;
2484                                         s2 = 0;
2485                                 }
2486                                 else {
2487                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2488                                                 sizeof(methodptr) * lm->class->index;
2489
2490                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2491                                 }
2492
2493                                 /* implicit null-pointer check */
2494                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2495                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2496                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2497                                 M_CALL(REG_ITMP3);
2498                                 break;
2499                         }
2500
2501                         /* generate method profiling code */
2502
2503                         PROFILE_CYCLE_START;
2504
2505                         /* store size of call code in replacement point */
2506
2507                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2508                         REPLACEMENT_POINT_FORGC_BUILTIN_RETURN(cd, iptr);
2509
2510                         /* store return value */
2511
2512                         switch (md->returntype.type) {
2513                         case TYPE_INT:
2514                         case TYPE_LNG:
2515                         case TYPE_ADR:
2516                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2517                                 M_INTMOVE(REG_RESULT, s1);
2518                                 emit_store_dst(jd, iptr, s1);
2519                                 break;
2520                         case TYPE_FLT:
2521                         case TYPE_DBL:
2522                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2523                                 M_FLTMOVE(REG_FRESULT, s1);
2524                                 emit_store_dst(jd, iptr, s1);
2525                                 break;
2526                         default:
2527                                 /* TYPE_VOID */
2528                                 break;
2529                         }
2530                         break;
2531
2532
2533                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2534
2535                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2536                                 /* object type cast-check */
2537
2538                                 classinfo *super;
2539                                 vftbl_t   *supervftbl;
2540                                 s4         superindex;
2541
2542                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2543                                         super      = NULL;
2544                                         superindex = 0;
2545                                         supervftbl = NULL;
2546                                 }
2547                                 else {
2548                                         super      = iptr->sx.s23.s3.c.cls;
2549                                         superindex = super->index;
2550                                         supervftbl = super->vftbl;
2551                                 }
2552
2553                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2554                                         CODEGEN_CRITICAL_SECTION_NEW;
2555
2556                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2557
2558                                 /* if class is not resolved, check which code to call */
2559
2560                                 if (super == NULL) {
2561                                         M_TEST(s1);
2562                                         emit_label_beq(cd, BRANCH_LABEL_1);
2563
2564                                         codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2565                                                                                   iptr->sx.s23.s3.c.ref, 0);
2566
2567                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2568                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2569                                         emit_label_beq(cd, BRANCH_LABEL_2);
2570                                 }
2571
2572                                 /* interface checkcast code */
2573
2574                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2575                                         if (super != NULL) {
2576                                                 M_TEST(s1);
2577                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2578                                         }
2579
2580                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2581
2582                                         if (super == NULL) {
2583                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_interface,
2584                                                                                           iptr->sx.s23.s3.c.ref,
2585                                                                                           0);
2586                                         }
2587
2588                                         M_ILD32(REG_ITMP3,
2589                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2590                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2591                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2592
2593                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2594                                                         OFFSET(vftbl_t, interfacetable[0]) -
2595                                                         superindex * sizeof(methodptr*));
2596                                         M_TEST(REG_ITMP3);
2597                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2598
2599                                         if (super == NULL)
2600                                                 emit_label_br(cd, BRANCH_LABEL_4);
2601                                         else
2602                                                 emit_label(cd, BRANCH_LABEL_3);
2603                                 }
2604
2605                                 /* class checkcast code */
2606
2607                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2608                                         if (super == NULL) {
2609                                                 emit_label(cd, BRANCH_LABEL_2);
2610                                         }
2611                                         else {
2612                                                 M_TEST(s1);
2613                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2614                                         }
2615
2616                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2617
2618                                         if (super == NULL) {
2619                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_class,
2620                                                                                           iptr->sx.s23.s3.c.ref,
2621                                                                                           0);
2622                                         }
2623
2624                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2625
2626                                         CODEGEN_CRITICAL_SECTION_START;
2627
2628                                         M_ILD32(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2629
2630                                         /*                                      if (s1 != REG_ITMP1) { */
2631                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2632                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2633                                         /*                                                                                              REG_ITMP1); */
2634                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2635                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2636                                         /*                                                                                              REG_ITMP3); */
2637                                         /*  #if defined(ENABLE_THREADS) */
2638                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2639                                         /*  #endif */
2640                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2641
2642                                         /*                                      } else { */
2643
2644                                         M_ILD32(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
2645                                         M_ISUB(REG_ITMP3, REG_ITMP2);
2646                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2647                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2648                                         /*                                      } */
2649
2650                                         CODEGEN_CRITICAL_SECTION_END;
2651
2652                                         M_ICMP(REG_ITMP3, REG_ITMP2);
2653                                         emit_classcast_check(cd, iptr, BRANCH_UGT, REG_ITMP3, s1);
2654
2655                                         if (super != NULL)
2656                                                 emit_label(cd, BRANCH_LABEL_5);
2657                                 }
2658
2659                                 if (super == NULL) {
2660                                         emit_label(cd, BRANCH_LABEL_1);
2661                                         emit_label(cd, BRANCH_LABEL_4);
2662                                 }
2663
2664                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2665                         }
2666                         else {
2667                                 /* array type cast-check */
2668
2669                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2670                                 M_INTMOVE(s1, REG_A0);
2671
2672                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2673                                         codegen_add_patch_ref(cd, PATCHER_builtin_arraycheckcast,
2674                                                                                   iptr->sx.s23.s3.c.ref, 0);
2675                                 }
2676
2677                                 M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2678                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2679                                 M_CALL(REG_ITMP1);
2680
2681                                 /* s1 may have been destroyed over the function call */
2682                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2683                                 M_TEST(REG_RESULT);
2684                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2685
2686                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2687                         }
2688
2689                         M_INTMOVE(s1, d);
2690                         emit_store_dst(jd, iptr, d);
2691                         break;
2692
2693                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2694
2695                         {
2696                         classinfo *super;
2697                         vftbl_t   *supervftbl;
2698                         s4         superindex;
2699
2700                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2701                                 super      = NULL;
2702                                 superindex = 0;
2703                                 supervftbl = NULL;
2704                         }
2705                         else {
2706                                 super      = iptr->sx.s23.s3.c.cls;
2707                                 superindex = super->index;
2708                                 supervftbl = super->vftbl;
2709                         }
2710
2711                         if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2712                                 CODEGEN_CRITICAL_SECTION_NEW;
2713
2714                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2715                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2716
2717                         if (s1 == d) {
2718                                 M_INTMOVE(s1, REG_ITMP1);
2719                                 s1 = REG_ITMP1;
2720                         }
2721
2722                         M_CLR(d);
2723
2724                         /* if class is not resolved, check which code to call */
2725
2726                         if (super == NULL) {
2727                                 M_TEST(s1);
2728                                 emit_label_beq(cd, BRANCH_LABEL_1);
2729
2730                                 codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2731                                                                           iptr->sx.s23.s3.c.ref, 0);
2732
2733                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2734                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2735                                 emit_label_beq(cd, BRANCH_LABEL_2);
2736                         }
2737
2738                         /* interface instanceof code */
2739
2740                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2741                                 if (super != NULL) {
2742                                         M_TEST(s1);
2743                                         emit_label_beq(cd, BRANCH_LABEL_3);
2744                                 }
2745
2746                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2747
2748                                 if (super == NULL) {
2749                                         codegen_add_patch_ref(cd, PATCHER_instanceof_interface,
2750                                                                                   iptr->sx.s23.s3.c.ref, 0);
2751                                 }
2752
2753                                 M_ILD32(REG_ITMP3,
2754                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2755                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2756
2757                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2758
2759                                 M_BLE(a);
2760                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2761                                                 OFFSET(vftbl_t, interfacetable[0]) -
2762                                                 superindex * sizeof(methodptr*));
2763                                 M_TEST(REG_ITMP1);
2764                                 M_SETNE(d);
2765
2766                                 if (super == NULL)
2767                                         emit_label_br(cd, BRANCH_LABEL_4);
2768                                 else
2769                                         emit_label(cd, BRANCH_LABEL_3);
2770                         }
2771
2772                         /* class instanceof code */
2773
2774                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2775                                 if (super == NULL) {
2776                                         emit_label(cd, BRANCH_LABEL_2);
2777                                 }
2778                                 else {
2779                                         M_TEST(s1);
2780                                         emit_label_beq(cd, BRANCH_LABEL_5);
2781                                 }
2782
2783                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2784
2785                                 if (super == NULL) {
2786                                         codegen_add_patch_ref(cd, PATCHER_instanceof_class,
2787                                                                                   iptr->sx.s23.s3.c.ref, 0);
2788                                 }
2789
2790                                 M_MOV_IMM(supervftbl, REG_ITMP2);
2791
2792                                 CODEGEN_CRITICAL_SECTION_START;
2793
2794                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
2795                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2796                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2797
2798                                 CODEGEN_CRITICAL_SECTION_END;
2799
2800                                 M_ISUB(REG_ITMP2, REG_ITMP1);
2801                                 M_CLR(d); /* may be REG_ITMP2 */
2802                                 M_ICMP(REG_ITMP3, REG_ITMP1);
2803                                 M_SETULE(d);
2804
2805                                 if (super != NULL)
2806                                         emit_label(cd, BRANCH_LABEL_5);
2807                         }
2808
2809                         if (super == NULL) {
2810                                 emit_label(cd, BRANCH_LABEL_1);
2811                                 emit_label(cd, BRANCH_LABEL_4);
2812                         }
2813
2814                         emit_store_dst(jd, iptr, d);
2815                         }
2816                         break;
2817
2818                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2819
2820                         /* check for negative sizes and copy sizes to stack if necessary  */
2821
2822                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2823
2824                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2825
2826                                 /* copy SAVEDVAR sizes to stack */
2827                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2828
2829                                 /* Already Preallocated? */
2830                                 if (!(var->flags & PREALLOC)) {
2831                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2832                                         M_LST(s2, REG_SP, s1 * 8);
2833                                 }
2834                         }
2835
2836                         /* is a patcher function set? */
2837
2838                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2839                                 codegen_add_patch_ref(cd, PATCHER_builtin_multianewarray,
2840                                                                           iptr->sx.s23.s3.c.ref, 0);
2841                         }
2842
2843                         /* a0 = dimension count */
2844
2845                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2846
2847                         /* a1 = classinfo */
2848
2849                         M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2850
2851                         /* a2 = pointer to dimensions = stack pointer */
2852
2853                         M_MOV(REG_SP, REG_A2);
2854
2855                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2856                         M_CALL(REG_ITMP1);
2857
2858                         /* check for exception before result assignment */
2859
2860                         emit_exception_check(cd, iptr);
2861
2862                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2863                         M_INTMOVE(REG_RESULT, s1);
2864                         emit_store_dst(jd, iptr, s1);
2865                         break;
2866
2867                 default:
2868                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2869                                                                                    iptr->opc);
2870                         return false;
2871         } /* switch */
2872
2873         } /* for instruction */
2874
2875         MCODECHECK(512); /* XXX require a lower number? */
2876
2877         /* At the end of a basic block we may have to append some nops,
2878            because the patcher stub calling code might be longer than the
2879            actual instruction. So codepatching does not change the
2880            following block unintentionally. */
2881
2882         if (cd->mcodeptr < cd->lastmcodeptr) {
2883                 while (cd->mcodeptr < cd->lastmcodeptr) {
2884                         M_NOP;
2885                 }
2886         }
2887
2888         } /* if (bptr -> flags >= BBREACHED) */
2889         } /* for basic block */
2890
2891         dseg_createlinenumbertable(cd);
2892
2893         /* generate stubs */
2894
2895         emit_patcher_stubs(jd);
2896
2897         /* everything's ok */
2898
2899         return true;
2900 }
2901
2902
2903 /* codegen_emit_stub_compiler **************************************************
2904
2905    Emit a stub routine which calls the compiler.
2906
2907 *******************************************************************************/
2908
2909 void codegen_emit_stub_compiler(jitdata *jd)
2910 {
2911         methodinfo  *m;
2912         codegendata *cd;
2913
2914         /* get required compiler data */
2915
2916         m  = jd->m;
2917         cd = jd->cd;
2918
2919         /* code for the stub */
2920
2921         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
2922         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
2923         M_JMP(REG_ITMP3);
2924 }
2925
2926
2927 /* codegen_emit_stub_builtin ***************************************************
2928
2929    Creates a stub routine which calls a builtin function.
2930
2931 *******************************************************************************/
2932
2933 void codegen_emit_stub_builtin(jitdata *jd, builtintable_entry *bte)
2934 {
2935         codeinfo    *code;
2936         codegendata *cd;
2937         methoddesc  *md;
2938         s4           i, j;
2939         s4           s1, disp;
2940
2941         /* get required compiler data */
2942
2943         code = jd->code;
2944         cd   = jd->cd;
2945
2946         md = bte->md;
2947
2948         /* calculate stack frame size */
2949
2950         cd->stackframesize =
2951                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
2952                 md->paramcount +                          /* saved argument registers */
2953                 1;                                                    /* return value */
2954
2955         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
2956
2957         /* create method header */
2958
2959         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2960         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2961         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
2962         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2963         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2964         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2965         (void) dseg_addlinenumbertablesize(cd);
2966         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
2967
2968         /* generate stub code */
2969
2970         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2971
2972 #if defined(ENABLE_GC_CACAO)
2973         /* Save callee saved integer registers in stackframeinfo (GC may
2974            need to recover them during a collection). */
2975
2976         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
2977                 OFFSET(stackframeinfo, intregs);
2978
2979         for (i = 0; i < INT_SAV_CNT; i++)
2980                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2981 #endif
2982
2983         /* save integer and float argument registers */
2984
2985         for (i = 0, j = 0; i < md->paramcount; i++) {
2986                 if (!md->params[i].inmemory) {
2987                         s1 = md->params[i].regoff;
2988
2989                         switch (md->paramtypes[i].type) {
2990                         case TYPE_INT:
2991                         case TYPE_LNG:
2992                         case TYPE_ADR:
2993                                 M_LST(s1, REG_SP, j * 8);
2994                                 break;
2995                         case TYPE_FLT:
2996                         case TYPE_DBL:
2997                                 M_DST(s1, REG_SP, j * 8);
2998                                 break;
2999                         }
3000
3001                         j++;
3002                 }
3003         }
3004
3005         /* create dynamic stack info */
3006
3007         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3008         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3009         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3010         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3011         M_MOV_IMM(codegen_stub_builtin_enter, REG_ITMP1);
3012         M_CALL(REG_ITMP1);
3013
3014         /* restore integer and float argument registers */
3015
3016         for (i = 0, j = 0; i < md->paramcount; i++) {
3017                 if (!md->params[i].inmemory) {
3018                         s1 = md->params[i].regoff;
3019
3020                         switch (md->paramtypes[i].type) {
3021                         case TYPE_INT:
3022                         case TYPE_LNG:
3023                         case TYPE_ADR:
3024                                 M_LLD(s1, REG_SP, j * 8);
3025                                 break;
3026                         case TYPE_FLT:
3027                         case TYPE_DBL:
3028                                 M_DLD(s1, REG_SP, j * 8);
3029                                 break;
3030                         }
3031
3032                         j++;
3033                 }
3034         }
3035
3036         /* call the builtin function */
3037
3038         M_MOV_IMM(bte->fp, REG_ITMP3);
3039         M_CALL(REG_ITMP3);
3040
3041         /* save return value */
3042
3043         switch (md->returntype.type) {
3044         case TYPE_INT:
3045         case TYPE_LNG:
3046         case TYPE_ADR:
3047                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3048                 break;
3049         case TYPE_FLT:
3050         case TYPE_DBL:
3051                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3052                 break;
3053         case TYPE_VOID:
3054                 break;
3055         }
3056
3057         /* remove native stackframe info */
3058
3059         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3060         M_MOV_IMM(codegen_stub_builtin_exit, REG_ITMP1);
3061         M_CALL(REG_ITMP1);
3062
3063         /* restore return value */
3064
3065         switch (md->returntype.type) {
3066         case TYPE_INT:
3067         case TYPE_LNG:
3068         case TYPE_ADR:
3069                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3070                 break;
3071         case TYPE_FLT:
3072         case TYPE_DBL:
3073                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3074                 break;
3075         case TYPE_VOID:
3076                 break;
3077         }
3078
3079 #if defined(ENABLE_GC_CACAO)
3080         /* Restore callee saved integer registers from stackframeinfo (GC
3081            might have modified them during a collection). */
3082
3083         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3084                 OFFSET(stackframeinfo, intregs);
3085
3086         for (i = 0; i < INT_SAV_CNT; i++)
3087                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3088 #endif
3089
3090         /* remove stackframe */
3091
3092         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3093         M_RET;
3094 }
3095
3096
3097 /* codegen_emit_stub_native ****************************************************
3098
3099    Emits a stub routine which calls a native method.
3100
3101 *******************************************************************************/
3102
3103 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f)
3104 {
3105         methodinfo  *m;
3106         codeinfo    *code;
3107         codegendata *cd;
3108         methoddesc  *md;
3109         s4           nativeparams;
3110         s4           i, j;
3111         s4           t;
3112         s4           s1, s2, disp;
3113
3114         /* get required compiler data */
3115
3116         m    = jd->m;
3117         code = jd->code;
3118         cd   = jd->cd;
3119
3120         /* initialize variables */
3121
3122         md = m->parseddesc;
3123         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
3124
3125         /* calculate stack frame size */
3126
3127         cd->stackframesize =
3128                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
3129                 sizeof(localref_table) / SIZEOF_VOID_P +
3130                 md->paramcount +
3131                 1 +                       /* functionptr, TODO: store in data segment */
3132                 nmd->memuse;
3133
3134         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
3135
3136         /* create method header */
3137
3138         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
3139         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
3140         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
3141         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
3142         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
3143         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
3144         (void) dseg_addlinenumbertablesize(cd);
3145         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
3146
3147 #if defined(ENABLE_PROFILING)
3148         /* generate native method profiling code */
3149
3150         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3151                 /* count frequency */
3152
3153                 M_MOV_IMM(code, REG_ITMP3);
3154                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3155         }
3156 #endif
3157
3158         /* generate stub code */
3159
3160         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3161
3162 #if !defined(NDEBUG)
3163         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3164                 emit_verbosecall_enter(jd);
3165 #endif
3166
3167         /* get function address (this must happen before the stackframeinfo) */
3168
3169 #if !defined(WITH_STATIC_CLASSPATH)
3170         if (f == NULL)
3171                 codegen_add_patch_ref(cd, PATCHER_resolve_native, m, 0);
3172 #endif
3173
3174         M_MOV_IMM(f, REG_ITMP3);
3175
3176 #if defined(ENABLE_GC_CACAO)
3177         /* Save callee saved integer registers in stackframeinfo (GC may
3178            need to recover them during a collection). */
3179
3180         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3181                 OFFSET(stackframeinfo, intregs);
3182
3183         for (i = 0; i < INT_SAV_CNT; i++)
3184                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3185 #endif
3186
3187         /* save integer and float argument registers */
3188
3189         for (i = 0; i < md->paramcount; i++) {
3190                 if (!md->params[i].inmemory) {
3191                         s1 = md->params[i].regoff;
3192
3193                         switch (md->paramtypes[i].type) {
3194                         case TYPE_INT:
3195                         case TYPE_LNG:
3196                         case TYPE_ADR:
3197                                 M_LST(s1, REG_SP, i * 8);
3198                                 break;
3199                         case TYPE_FLT:
3200                         case TYPE_DBL:
3201                                 M_DST(s1, REG_SP, i * 8);
3202                                 break;
3203                         }
3204                 }
3205         }
3206
3207         M_AST(REG_ITMP3, REG_SP, md->paramcount * 8);
3208
3209         /* create dynamic stack info */
3210
3211         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3212         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3213         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3214         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3215         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3216         M_CALL(REG_ITMP1);
3217
3218         /* restore integer and float argument registers */
3219
3220         for (i = 0; i < md->paramcount; i++) {
3221                 if (!md->params[i].inmemory) {
3222                         s1 = md->params[i].regoff;
3223
3224                         switch (md->paramtypes[i].type) {
3225                         case TYPE_INT:
3226                         case TYPE_LNG:
3227                         case TYPE_ADR:
3228                                 M_LLD(s1, REG_SP, i * 8);
3229                                 break;
3230                         case TYPE_FLT:
3231                         case TYPE_DBL:
3232                                 M_DLD(s1, REG_SP, i * 8);
3233                                 break;
3234                         }
3235                 }
3236         }
3237
3238         M_ALD(REG_ITMP3, REG_SP, md->paramcount * 8);
3239
3240         /* copy or spill arguments to new locations */
3241
3242         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3243                 t  = md->paramtypes[i].type;
3244                 s2 = nmd->params[j].regoff;
3245
3246                 if (IS_INT_LNG_TYPE(t)) {
3247                         if (!md->params[i].inmemory) {
3248                                 s1 = md->params[i].regoff;
3249
3250                                 if (!nmd->params[j].inmemory)
3251                                         M_INTMOVE(s1, s2);
3252                                 else
3253                                         M_LST(s1, REG_SP, s2);
3254                         }
3255                         else {
3256                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3257                                 M_LLD(REG_ITMP1, REG_SP, s1);
3258                                 M_LST(REG_ITMP1, REG_SP, s2);
3259                         }
3260                 }
3261                 else {
3262                         /* We only copy spilled float arguments, as the float
3263                            argument registers keep unchanged. */
3264
3265                         if (md->params[i].inmemory) {
3266                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3267
3268                                 if (IS_2_WORD_TYPE(t)) {
3269                                         M_DLD(REG_FTMP1, REG_SP, s1);
3270                                         M_DST(REG_FTMP1, REG_SP, s2);
3271                                 }
3272                                 else {
3273                                         M_FLD(REG_FTMP1, REG_SP, s1);
3274                                         M_FST(REG_FTMP1, REG_SP, s2);
3275                                 }
3276                         }
3277                 }
3278         }
3279
3280         /* put class into second argument register */
3281
3282         if (m->flags & ACC_STATIC)
3283                 M_MOV_IMM(m->class, REG_A1);
3284
3285         /* put env into first argument register */
3286
3287         M_MOV_IMM(_Jv_env, REG_A0);
3288
3289         /* do the native function call */
3290
3291         M_CALL(REG_ITMP3);
3292
3293         /* save return value */
3294
3295         switch (md->returntype.type) {
3296         case TYPE_INT:
3297         case TYPE_LNG:
3298         case TYPE_ADR:
3299                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3300                 break;
3301         case TYPE_FLT:
3302         case TYPE_DBL:
3303                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3304                 break;
3305         case TYPE_VOID:
3306                 break;
3307         }
3308
3309 #if !defined(NDEBUG)
3310         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3311                 emit_verbosecall_exit(jd);
3312 #endif
3313
3314         /* remove native stackframe info */
3315
3316         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3317         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3318         M_CALL(REG_ITMP1);
3319         M_MOV(REG_RESULT, REG_ITMP3);
3320
3321         /* restore return value */
3322
3323         switch (md->returntype.type) {
3324         case TYPE_INT:
3325         case TYPE_LNG:
3326         case TYPE_ADR:
3327                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3328                 break;
3329         case TYPE_FLT:
3330         case TYPE_DBL:
3331                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3332                 break;
3333         case TYPE_VOID:
3334                 break;
3335         }
3336
3337 #if defined(ENABLE_GC_CACAO)
3338         /* Restore callee saved integer registers from stackframeinfo (GC
3339            might have modified them during a collection). */
3340
3341         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3342                 OFFSET(stackframeinfo, intregs);
3343
3344         for (i = 0; i < INT_SAV_CNT; i++)
3345                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3346 #endif
3347
3348         /* remove stackframe */
3349
3350         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3351
3352         /* test for exception */
3353
3354         M_TEST(REG_ITMP3);
3355         M_BNE(1);
3356         M_RET;
3357
3358         /* handle exception */
3359
3360         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3361         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3362         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3363
3364         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3365         M_JMP(REG_ITMP3);
3366
3367         /* generate patcher stubs */
3368
3369         emit_patcher_stubs(jd);
3370 }
3371
3372
3373 /*
3374  * These are local overrides for various environment variables in Emacs.
3375  * Please do not remove this and leave it at the end of the file, where
3376  * Emacs will automagically detect them.
3377  * ---------------------------------------------------------------------
3378  * Local variables:
3379  * mode: c
3380  * indent-tabs-mode: t
3381  * c-basic-offset: 4
3382  * tab-width: 4
3383  * End:
3384  * vim:noexpandtab:sw=4:ts=4:
3385  */