src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    $Id: codegen.c 8299 2007-08-13 08:41:18Z michi $
  26
  27 */
  28
  29
  30 #include "config.h"
  31
  32 #include <assert.h>
  33 #include <stdio.h>
  34
  35 #include "vm/types.h"
  36
  37 #include "md-abi.h"
  38
  39 #include "vm/jit/x86_64/arch.h"
  40 #include "vm/jit/x86_64/codegen.h"
  41 #include "vm/jit/x86_64/emit.h"
  42
  43 #include "mm/memory.h"
  44
  45 #include "native/jni.h"
  46 #include "native/localref.h"
  47 #include "native/native.h"
  48
  49 #include "threads/lock-common.h"
  50
  51 #include "vm/builtin.h"
  52 #include "vm/exceptions.h"
  53 #include "vm/global.h"
  54 #include "vm/stringlocal.h"
  55 #include "vm/vm.h"
  56
  57 #include "vm/jit/abi.h"
  58 #include "vm/jit/asmpart.h"
  59 #include "vm/jit/codegen-common.h"
  60 #include "vm/jit/dseg.h"
  61 #include "vm/jit/emit-common.h"
  62 #include "vm/jit/jit.h"
  63 #include "vm/jit/methodheader.h"
  64 #include "vm/jit/parse.h"
  65 #include "vm/jit/patcher.h"
  66 #include "vm/jit/reg.h"
  67 #include "vm/jit/replace.h"
  68 #include "vm/jit/stacktrace.h"
  69
  70 #if defined(ENABLE_LSRA)
  71 # include "vm/jit/allocator/lsra.h"
  72 #endif
  73
  74 #include "vmcore/loader.h"
  75 #include "vmcore/options.h"
  76 #include "vmcore/statistics.h"
  77
  78
  79 /* codegen_emit ****************************************************************
  80
  81    Generates machine code.
  82
  83 *******************************************************************************/
  84
  85 bool codegen_emit(jitdata *jd)
  86 {
  87         methodinfo         *m;
  88         codeinfo           *code;
  89         codegendata        *cd;
  90         registerdata       *rd;
  91         s4                  len, s1, s2, s3, d, disp;
  92         u2                  currentline;
  93         ptrint              a;
  94         varinfo            *var, *dst;
  95         basicblock         *bptr;
  96         instruction        *iptr;
  97         exception_entry    *ex;
  98         constant_classref  *cr;
  99         unresolved_class   *uc;
 100         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 101         unresolved_method  *um;
 102         builtintable_entry *bte;
 103         methoddesc         *md;
 104         fieldinfo          *fi;
 105         unresolved_field   *uf;
 106         s4                  fieldtype;
 107         s4                 varindex;
 108
 109         /* get required compiler data */
 110
 111         m    = jd->m;
 112         code = jd->code;
 113         cd   = jd->cd;
 114         rd   = jd->rd;
 115
 116         /* prevent compiler warnings */
 117
 118         d   = 0;
 119         lm  = NULL;
 120         um  = NULL;
 121         bte = NULL;
 122
 123         {
 124         s4 i, p, t, l;
 125         s4 savedregs_num;
 126
 127         savedregs_num = 0;
 128
 129         /* space to save used callee saved registers */
 130
 131         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 132         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 133
 134         cd->stackframesize = rd->memuse + savedregs_num;
 135
 136 #if defined(ENABLE_THREADS)
 137         /* space to save argument of monitor_enter */
 138
 139         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 140                 cd->stackframesize++;
 141 #endif
 142
 143         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 144            native code e.g. libc or jni (alignment problems with
 145            movaps). */
 146
 147         if (!jd->isleafmethod || opt_verbosecall)
 148                 cd->stackframesize |= 0x1;
 149
 150         /* create method header */
 151
 152         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 153         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 154
 155 #if defined(ENABLE_THREADS)
 156         /* IsSync contains the offset relative to the stack pointer for the
 157            argument of monitor_exit used in the exception handler. Since the
 158            offset could be zero and give a wrong meaning of the flag it is
 159            offset by one.
 160         */
 161
 162         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 163                 (void) dseg_add_unique_s4(cd, (rd->memuse + 1) * 8); /* IsSync        */
 164         else
 165 #endif
 166                 (void) dseg_add_unique_s4(cd, 0);                  /* IsSync          */
 167
 168         (void) dseg_add_unique_s4(cd, jd->isleafmethod);               /* IsLeaf  */
 169         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 170         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 171
 172         (void) dseg_addlinenumbertablesize(cd);
 173
 174         (void) dseg_add_unique_s4(cd, jd->exceptiontablelength); /* ExTableSize   */
 175
 176         /* create exception table */
 177
 178         for (ex = jd->exceptiontable; ex != NULL; ex = ex->down) {
 179                 dseg_add_target(cd, ex->start);
 180                 dseg_add_target(cd, ex->end);
 181                 dseg_add_target(cd, ex->handler);
 182                 (void) dseg_add_unique_address(cd, ex->catchtype.any);
 183         }
 184
 185 #if defined(ENABLE_PROFILING)
 186         /* generate method profiling code */
 187
 188         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 189                 /* count frequency */
 190
 191                 M_MOV_IMM(code, REG_ITMP3);
 192                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 193
 194                 PROFILE_CYCLE_START;
 195         }
 196 #endif
 197
 198         /* create stack frame (if necessary) */
 199
 200         if (cd->stackframesize)
 201                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 202
 203         /* save used callee saved registers */
 204
 205         p = cd->stackframesize;
 206         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 207                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 208         }
 209         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 210                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 211         }
 212
 213         /* take arguments out of register or stack frame */
 214
 215         md = m->parseddesc;
 216
 217         for (p = 0, l = 0; p < md->paramcount; p++) {
 218                 t = md->paramtypes[p].type;
 219
 220                 varindex = jd->local_map[l * 5 + t];
 221
 222                 l++;
 223                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 224                         l++;
 225
 226                 if (varindex == UNUSED)
 227                         continue;
 228
 229                 var = VAR(varindex);
 230
 231                 s1 = md->params[p].regoff;
 232
 233                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 234                         if (!md->params[p].inmemory) {           /* register arguments    */
 235                                 if (!IS_INMEMORY(var->flags))
 236                                         M_INTMOVE(s1, var->vv.regoff);
 237                                 else
 238                                     M_LST(s1, REG_SP, var->vv.regoff);
 239                         }
 240                         else {                                 /* stack arguments       */
 241                                 if (!IS_INMEMORY(var->flags))
 242                                         /* + 8 for return address */
 243                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 244                                 else
 245                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 246                         }
 247                 }
 248                 else {                                     /* floating args         */
 249                         if (!md->params[p].inmemory) {           /* register arguments    */
 250                                 if (!IS_INMEMORY(var->flags))
 251                                         M_FLTMOVE(s1, var->vv.regoff);
 252                                 else
 253                                         M_DST(s1, REG_SP, var->vv.regoff);
 254                         }
 255                         else {                                 /* stack arguments       */
 256                                 if (!IS_INMEMORY(var->flags))
 257                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 258                                 else
 259                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 260                         }
 261                 }
 262         }
 263
 264         /* save monitorenter argument */
 265
 266 #if defined(ENABLE_THREADS)
 267         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 268                 /* stack offset for monitor argument */
 269
 270                 s1 = rd->memuse;
 271
 272                 if (opt_verbosecall) {
 273                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 274
 275                         for (p = 0; p < INT_ARG_CNT; p++)
 276                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 277
 278                         for (p = 0; p < FLT_ARG_CNT; p++)
 279                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 280
 281                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 282                 }
 283
 284                 /* decide which monitor enter function to call */
 285
 286                 if (m->flags & ACC_STATIC) {
 287                         M_MOV_IMM(&m->class->object.header, REG_A0);
 288                 }
 289                 else {
 290                         M_TEST(REG_A0);
 291                         M_BNE(8);
 292                         M_ALD_MEM(REG_A0, EXCEPTION_HARDWARE_NULLPOINTER);
 293                 }
 294
 295                 M_AST(REG_A0, REG_SP, s1 * 8);
 296                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 297                 M_CALL(REG_ITMP1);
 298
 299                 if (opt_verbosecall) {
 300                         for (p = 0; p < INT_ARG_CNT; p++)
 301                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 302
 303                         for (p = 0; p < FLT_ARG_CNT; p++)
 304                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 305
 306                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 307                 }
 308         }
 309 #endif
 310
 311 #if !defined(NDEBUG)
 312         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 313                 emit_verbosecall_enter(jd);
 314 #endif /* !defined(NDEBUG) */
 315
 316         }
 317
 318         /* end of header generation */
 319
 320         /* create replacement points */
 321
 322         REPLACEMENT_POINTS_INIT(cd, jd);
 323
 324         /* walk through all basic blocks */
 325
 326         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 327
 328                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 329
 330                 if (bptr->flags >= BBREACHED) {
 331
 332                 /* branch resolving */
 333
 334                 codegen_resolve_branchrefs(cd, bptr);
 335
 336                 /* handle replacement points */
 337
 338                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 339
 340                 /* copy interface registers to their destination */
 341
 342                 len = bptr->indepth;
 343                 MCODECHECK(512);
 344
 345 #if defined(ENABLE_PROFILING)
 346                 /* generate basicblock profiling code */
 347
 348                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 349                         /* count frequency */
 350
 351                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 352                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 353
 354                         /* if this is an exception handler, start profiling again */
 355
 356                         if (bptr->type == BBTYPE_EXH)
 357                                 PROFILE_CYCLE_START;
 358                 }
 359 #endif
 360
 361 #if defined(ENABLE_LSRA)
 362                 if (opt_lsra) {
 363                         while (len) {
 364                                 len--;
 365                                 src = bptr->invars[len];
 366                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 367                                         if (bptr->type == BBTYPE_EXH) {
 368 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 369                                                 if (!IS_INMEMORY(src->flags))
 370                                                         d= src->vv.regoff;
 371                                                 else
 372                                                         d=REG_ITMP1;
 373                                                 M_INTMOVE(REG_ITMP1, d);
 374                                                 emit_store(jd, NULL, src, d);
 375                                         }
 376                                 }
 377                         }
 378
 379                 } else {
 380 #endif
 381
 382                 while (len) {
 383                         len--;
 384                         var = VAR(bptr->invars[len]);
 385                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 386                                 if (bptr->type == BBTYPE_EXH) {
 387                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 388                                         M_INTMOVE(REG_ITMP1, d);
 389                                         emit_store(jd, NULL, var, d);
 390                                 }
 391                         }
 392                         else {
 393                                 assert((var->flags & INOUT));
 394                         }
 395                 }
 396 #if defined(ENABLE_LSRA)
 397                 }
 398 #endif
 399                 /* walk through all instructions */
 400
 401                 len = bptr->icount;
 402                 currentline = 0;
 403
 404                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 405                         if (iptr->line != currentline) {
 406                                 dseg_addlinenumber(cd, iptr->line);
 407                                 currentline = iptr->line;
 408                         }
 409
 410                         MCODECHECK(1024);                         /* 1KB should be enough */
 411
 412                 switch (iptr->opc) {
 413                 case ICMD_NOP:        /* ...  ==> ...                                 */
 414                 case ICMD_POP:        /* ..., value  ==> ...                          */
 415                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 416                         break;
 417
 418                 case ICMD_INLINE_START:
 419
 420                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 421                         break;
 422
 423                 case ICMD_INLINE_BODY:
 424
 425                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 426                         dseg_addlinenumber_inline_start(cd, iptr);
 427                         dseg_addlinenumber(cd, iptr->line);
 428                         break;
 429
 430                 case ICMD_INLINE_END:
 431
 432                         dseg_addlinenumber_inline_end(cd, iptr);
 433                         dseg_addlinenumber(cd, iptr->line);
 434                         break;
 435
 436                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 437
 438                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 439                         emit_nullpointer_check(cd, iptr, s1);
 440                         break;
 441
 442                 /* constant operations ************************************************/
 443
 444                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 445
 446                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 447                         ICONST(d, iptr->sx.val.i);
 448                         emit_store_dst(jd, iptr, d);
 449                         break;
 450
 451                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 452
 453                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 454                         LCONST(d, iptr->sx.val.l);
 455                         emit_store_dst(jd, iptr, d);
 456                         break;
 457
 458                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 459
 460                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 461                         disp = dseg_add_float(cd, iptr->sx.val.f);
 462                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 463                         emit_store_dst(jd, iptr, d);
 464                         break;
 465
 466                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 467
 468                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 469                         disp = dseg_add_double(cd, iptr->sx.val.d);
 470                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 471                         emit_store_dst(jd, iptr, d);
 472                         break;
 473
 474                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 475
 476                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 477
 478                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 479                                 cr = iptr->sx.val.c.ref;
 480
 481 /*                              PROFILE_CYCLE_STOP; */
 482
 483                                 codegen_add_patch_ref(cd, PATCHER_aconst, cr, 0);
 484
 485 /*                              PROFILE_CYCLE_START; */
 486
 487                                 M_MOV_IMM(NULL, d);
 488
 489                         } else {
 490                                 if (iptr->sx.val.anyptr == 0)
 491                                         M_CLR(d);
 492                                 else
 493                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 494                         }
 495                         emit_store_dst(jd, iptr, d);
 496                         break;
 497
 498
 499                 /* load/store/copy/move operations ************************************/
 500
 501                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 502                 case ICMD_ALOAD:      /* s1 = local variable                          */
 503                 case ICMD_LLOAD:
 504                 case ICMD_FLOAD:
 505                 case ICMD_DLOAD:
 506                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 507                 case ICMD_LSTORE:
 508                 case ICMD_FSTORE:
 509                 case ICMD_DSTORE:
 510                 case ICMD_COPY:
 511                 case ICMD_MOVE:
 512
 513                         emit_copy(jd, iptr);
 514                         break;
 515
 516                 case ICMD_ASTORE:
 517                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 518                                 emit_copy(jd, iptr);
 519                         break;
 520
 521                 /* integer operations *************************************************/
 522
 523                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 524
 525                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 526                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 527                         M_INTMOVE(s1, d);
 528                         M_INEG(d);
 529                         emit_store_dst(jd, iptr, d);
 530                         break;
 531
 532                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 533
 534                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 535                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 536                         M_INTMOVE(s1, d);
 537                         M_LNEG(d);
 538                         emit_store_dst(jd, iptr, d);
 539                         break;
 540
 541                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 542
 543                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 544                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 545                         M_ISEXT(s1, d);
 546                         emit_store_dst(jd, iptr, d);
 547                         break;
 548
 549                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 550
 551                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 552                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 553                         M_IMOV(s1, d);
 554                         emit_store_dst(jd, iptr, d);
 555                         break;
 556
 557                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 558
 559                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 560                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 561                         M_BSEXT(s1, d);
 562                         emit_store_dst(jd, iptr, d);
 563                         break;
 564
 565                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 566
 567                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 568                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 569                         M_CZEXT(s1, d);
 570                         emit_store_dst(jd, iptr, d);
 571                         break;
 572
 573                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 574
 575                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 576                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 577                         M_SSEXT(s1, d);
 578                         emit_store_dst(jd, iptr, d);
 579                         break;
 580
 581
 582                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 583
 584                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 585                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 586                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 587                         if (s2 == d)
 588                                 M_IADD(s1, d);
 589                         else {
 590                                 M_INTMOVE(s1, d);
 591                                 M_IADD(s2, d);
 592                         }
 593                         emit_store_dst(jd, iptr, d);
 594                         break;
 595
 596                 case ICMD_IINC:
 597                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 598                                       /* sx.val.i = constant                             */
 599
 600                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 601                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 602
 603                         /* Using inc and dec is not faster than add (tested with
 604                            sieve). */
 605
 606                         M_INTMOVE(s1, d);
 607                         M_IADD_IMM(iptr->sx.val.i, d);
 608                         emit_store_dst(jd, iptr, d);
 609                         break;
 610
 611                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 612
 613                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 614                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 615                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 616                         if (s2 == d)
 617                                 M_LADD(s1, d);
 618                         else {
 619                                 M_INTMOVE(s1, d);
 620                                 M_LADD(s2, d);
 621                         }
 622                         emit_store_dst(jd, iptr, d);
 623                         break;
 624
 625                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 626                                       /* sx.val.l = constant                             */
 627
 628                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 629                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 630                         M_INTMOVE(s1, d);
 631                         if (IS_IMM32(iptr->sx.val.l))
 632                                 M_LADD_IMM(iptr->sx.val.l, d);
 633                         else {
 634                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 635                                 M_LADD(REG_ITMP2, d);
 636                         }
 637                         emit_store_dst(jd, iptr, d);
 638                         break;
 639
 640                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 641
 642                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 643                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 644                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 645                         if (s2 == d) {
 646                                 M_INTMOVE(s1, REG_ITMP1);
 647                                 M_ISUB(s2, REG_ITMP1);
 648                                 M_INTMOVE(REG_ITMP1, d);
 649                         } else {
 650                                 M_INTMOVE(s1, d);
 651                                 M_ISUB(s2, d);
 652                         }
 653                         emit_store_dst(jd, iptr, d);
 654                         break;
 655
 656                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 657                                       /* sx.val.i = constant                             */
 658
 659                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 660                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 661                         M_INTMOVE(s1, d);
 662                         M_ISUB_IMM(iptr->sx.val.i, d);
 663                         emit_store_dst(jd, iptr, d);
 664                         break;
 665
 666                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 667
 668                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 669                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 670                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 671                         if (s2 == d) {
 672                                 M_INTMOVE(s1, REG_ITMP1);
 673                                 M_LSUB(s2, REG_ITMP1);
 674                                 M_INTMOVE(REG_ITMP1, d);
 675                         } else {
 676                                 M_INTMOVE(s1, d);
 677                                 M_LSUB(s2, d);
 678                         }
 679                         emit_store_dst(jd, iptr, d);
 680                         break;
 681
 682                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 683                                       /* sx.val.l = constant                             */
 684
 685                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 686                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 687                         M_INTMOVE(s1, d);
 688                         if (IS_IMM32(iptr->sx.val.l))
 689                                 M_LSUB_IMM(iptr->sx.val.l, d);
 690                         else {
 691                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 692                                 M_LSUB(REG_ITMP2, d);
 693                         }
 694                         emit_store_dst(jd, iptr, d);
 695                         break;
 696
 697                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 698
 699                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 700                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 701                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 702                         if (s2 == d)
 703                                 M_IMUL(s1, d);
 704                         else {
 705                                 M_INTMOVE(s1, d);
 706                                 M_IMUL(s2, d);
 707                         }
 708                         emit_store_dst(jd, iptr, d);
 709                         break;
 710
 711                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 712                                       /* sx.val.i = constant                             */
 713
 714                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 715                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 716                         if (iptr->sx.val.i == 2) {
 717                                 M_INTMOVE(s1, d);
 718                                 M_ISLL_IMM(1, d);
 719                         } else
 720                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 721                         emit_store_dst(jd, iptr, d);
 722                         break;
 723
 724                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 725
 726                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 727                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 728                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 729                         if (s2 == d)
 730                                 M_LMUL(s1, d);
 731                         else {
 732                                 M_INTMOVE(s1, d);
 733                                 M_LMUL(s2, d);
 734                         }
 735                         emit_store_dst(jd, iptr, d);
 736                         break;
 737
 738                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 739                                       /* sx.val.l = constant                             */
 740
 741                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 742                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 743                         if (IS_IMM32(iptr->sx.val.l))
 744                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 745                         else {
 746                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 747                                 M_INTMOVE(s1, d);
 748                                 M_LMUL(REG_ITMP2, d);
 749                         }
 750                         emit_store_dst(jd, iptr, d);
 751                         break;
 752
 753                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 754
 755                         s1 = emit_load_s1(jd, iptr, RAX);
 756                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 757                         d = codegen_reg_of_dst(jd, iptr, RAX);
 758
 759                         M_INTMOVE(s1, RAX);
 760                         M_INTMOVE(s2, REG_ITMP3);
 761                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 762
 763                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 764
 765                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 766                         M_BNE(4 + 6);
 767                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 768                         M_BEQ(1 + 3);                                          /* 6 bytes */
 769
 770                         emit_cltd(cd);                                         /* 1 byte  */
 771                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 772
 773                         M_INTMOVE(RAX, d);
 774                         emit_store_dst(jd, iptr, d);
 775                         dst = VAROP(iptr->dst);
 776                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 777                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 778                         break;
 779
 780                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 781
 782                         s1 = emit_load_s1(jd, iptr, RAX);
 783                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 784                         d = codegen_reg_of_dst(jd, iptr, RDX);
 785
 786                         M_INTMOVE(s1, RAX);
 787                         M_INTMOVE(s2, REG_ITMP3);
 788                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 789
 790                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 791
 792                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 793                         M_BNE(3 + 4 + 6);
 794                         M_CLR(RDX);                                            /* 3 bytes */
 795                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 796                         M_BEQ(1 + 3);                                          /* 6 bytes */
 797
 798                         emit_cltd(cd);                                         /* 1 byte  */
 799                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 800
 801                         M_INTMOVE(RDX, d);
 802                         emit_store_dst(jd, iptr, d);
 803                         dst = VAROP(iptr->dst);
 804                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 805                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 806                         break;
 807
 808                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 809                                       /* sx.val.i = constant                             */
 810
 811                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 812                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 813                         M_INTMOVE(s1, REG_ITMP1);
 814                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 815                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 816                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 817                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 818                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 819                         emit_store_dst(jd, iptr, d);
 820                         break;
 821
 822                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 823                                       /* sx.val.i = constant                             */
 824
 825                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 826                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 827                         M_INTMOVE(s1, REG_ITMP1);
 828                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 829                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 830                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 831                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 832                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 833                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 834                         emit_store_dst(jd, iptr, d);
 835                         break;
 836
 837
 838                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 839
 840                         s1 = emit_load_s1(jd, iptr, RAX);
 841                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 842                         d = codegen_reg_of_dst(jd, iptr, RAX);
 843
 844                         M_INTMOVE(s1, RAX);
 845                         M_INTMOVE(s2, REG_ITMP3);
 846                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 847
 848                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 849
 850                         /* check as described in jvm spec */
 851                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 852                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 853                         M_BNE(4 + 6);
 854                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 855                         M_BEQ(2 + 3);                                          /* 6 bytes */
 856
 857                         emit_cqto(cd);                                         /* 2 bytes */
 858                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 859
 860                         M_INTMOVE(RAX, d);
 861                         emit_store_dst(jd, iptr, d);
 862                         dst = VAROP(iptr->dst);
 863                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 864                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 865                         break;
 866
 867                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 868
 869                         s1 = emit_load_s1(jd, iptr, RAX);
 870                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 871                         d = codegen_reg_of_dst(jd, iptr, RDX);
 872
 873                         M_INTMOVE(s1, RAX);
 874                         M_INTMOVE(s2, REG_ITMP3);
 875                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 876
 877                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 878
 879                         /* check as described in jvm spec */
 880                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 881                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 882                         M_BNE(3 + 4 + 6);
 883                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 884                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 885                         M_BEQ(2 + 3);                                          /* 6 bytes */
 886
 887                         emit_cqto(cd);                                         /* 2 bytes */
 888                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 889
 890                         M_INTMOVE(RDX, d);
 891                         emit_store_dst(jd, iptr, d);
 892                         dst = VAROP(iptr->dst);
 893                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 894                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 895                         break;
 896
 897                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 898                                       /* sx.val.i = constant                             */
 899
 900                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 901                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 902                         M_INTMOVE(s1, REG_ITMP1);
 903                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 904                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 905                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 906                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 907                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 908                         emit_store_dst(jd, iptr, d);
 909                         break;
 910
 911                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 912                                       /* sx.val.l = constant                             */
 913
 914                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 915                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 916                         M_INTMOVE(s1, REG_ITMP1);
 917                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 918                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 919                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 920                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 921                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 922                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 923                         emit_store_dst(jd, iptr, d);
 924                         break;
 925
 926                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 927
 928                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 929                         emit_ishift(jd, SHIFT_SHL, iptr);
 930                         break;
 931
 932                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 933                                       /* sx.val.i = constant                             */
 934
 935                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 936                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 937                         M_INTMOVE(s1, d);
 938                         M_ISLL_IMM(iptr->sx.val.i, d);
 939                         emit_store_dst(jd, iptr, d);
 940                         break;
 941
 942                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 943
 944                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 945                         emit_ishift(jd, SHIFT_SAR, iptr);
 946                         break;
 947
 948                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 949                                       /* sx.val.i = constant                             */
 950
 951                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 952                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 953                         M_INTMOVE(s1, d);
 954                         M_ISRA_IMM(iptr->sx.val.i, d);
 955                         emit_store_dst(jd, iptr, d);
 956                         break;
 957
 958                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 959
 960                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 961                         emit_ishift(jd, SHIFT_SHR, iptr);
 962                         break;
 963
 964                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 965                                       /* sx.val.i = constant                             */
 966
 967                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 968                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 969                         M_INTMOVE(s1, d);
 970                         M_ISRL_IMM(iptr->sx.val.i, d);
 971                         emit_store_dst(jd, iptr, d);
 972                         break;
 973
 974                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 975
 976                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 977                         emit_lshift(jd, SHIFT_SHL, iptr);
 978                         break;
 979
 980         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 981                                           /* sx.val.i = constant                             */
 982
 983                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 984                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 985                         M_INTMOVE(s1, d);
 986                         M_LSLL_IMM(iptr->sx.val.i, d);
 987                         emit_store_dst(jd, iptr, d);
 988                         break;
 989
 990                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 991
 992                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 993                         emit_lshift(jd, SHIFT_SAR, iptr);
 994                         break;
 995
 996                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 997                                       /* sx.val.i = constant                             */
 998
 999                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1000                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1001                         M_INTMOVE(s1, d);
1002                         M_LSRA_IMM(iptr->sx.val.i, d);
1003                         emit_store_dst(jd, iptr, d);
1004                         break;
1005
1006                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1007
1008                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1009                         emit_lshift(jd, SHIFT_SHR, iptr);
1010                         break;
1011
1012                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1013                                       /* sx.val.l = constant                             */
1014
1015                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1016                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1017                         M_INTMOVE(s1, d);
1018                         M_LSRL_IMM(iptr->sx.val.i, d);
1019                         emit_store_dst(jd, iptr, d);
1020                         break;
1021
1022                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1023
1024                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1025                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1026                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1027                         if (s2 == d)
1028                                 M_IAND(s1, d);
1029                         else {
1030                                 M_INTMOVE(s1, d);
1031                                 M_IAND(s2, d);
1032                         }
1033                         emit_store_dst(jd, iptr, d);
1034                         break;
1035
1036                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1037                                       /* sx.val.i = constant                             */
1038
1039                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1040                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1041                         M_INTMOVE(s1, d);
1042                         M_IAND_IMM(iptr->sx.val.i, d);
1043                         emit_store_dst(jd, iptr, d);
1044                         break;
1045
1046                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1047
1048                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1049                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1050                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1051                         if (s2 == d)
1052                                 M_LAND(s1, d);
1053                         else {
1054                                 M_INTMOVE(s1, d);
1055                                 M_LAND(s2, d);
1056                         }
1057                         emit_store_dst(jd, iptr, d);
1058                         break;
1059
1060                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1061                                       /* sx.val.l = constant                             */
1062
1063                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1064                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1065                         M_INTMOVE(s1, d);
1066                         if (IS_IMM32(iptr->sx.val.l))
1067                                 M_LAND_IMM(iptr->sx.val.l, d);
1068                         else {
1069                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1070                                 M_LAND(REG_ITMP2, d);
1071                         }
1072                         emit_store_dst(jd, iptr, d);
1073                         break;
1074
1075                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1076
1077                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1078                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1079                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1080                         if (s2 == d)
1081                                 M_IOR(s1, d);
1082                         else {
1083                                 M_INTMOVE(s1, d);
1084                                 M_IOR(s2, d);
1085                         }
1086                         emit_store_dst(jd, iptr, d);
1087                         break;
1088
1089                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1090                                       /* sx.val.i = constant                             */
1091
1092                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1093                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1094                         M_INTMOVE(s1, d);
1095                         M_IOR_IMM(iptr->sx.val.i, d);
1096                         emit_store_dst(jd, iptr, d);
1097                         break;
1098
1099                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1100
1101                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1102                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1103                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1104                         if (s2 == d)
1105                                 M_LOR(s1, d);
1106                         else {
1107                                 M_INTMOVE(s1, d);
1108                                 M_LOR(s2, d);
1109                         }
1110                         emit_store_dst(jd, iptr, d);
1111                         break;
1112
1113                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1114                                       /* sx.val.l = constant                             */
1115
1116                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1117                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1118                         M_INTMOVE(s1, d);
1119                         if (IS_IMM32(iptr->sx.val.l))
1120                                 M_LOR_IMM(iptr->sx.val.l, d);
1121                         else {
1122                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1123                                 M_LOR(REG_ITMP2, d);
1124                         }
1125                         emit_store_dst(jd, iptr, d);
1126                         break;
1127
1128                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1129
1130                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1131                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1132                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1133                         if (s2 == d)
1134                                 M_IXOR(s1, d);
1135                         else {
1136                                 M_INTMOVE(s1, d);
1137                                 M_IXOR(s2, d);
1138                         }
1139                         emit_store_dst(jd, iptr, d);
1140                         break;
1141
1142                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1143                                       /* sx.val.i = constant                             */
1144
1145                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1146                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1147                         M_INTMOVE(s1, d);
1148                         M_IXOR_IMM(iptr->sx.val.i, d);
1149                         emit_store_dst(jd, iptr, d);
1150                         break;
1151
1152                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1153
1154                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1155                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1156                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1157                         if (s2 == d)
1158                                 M_LXOR(s1, d);
1159                         else {
1160                                 M_INTMOVE(s1, d);
1161                                 M_LXOR(s2, d);
1162                         }
1163                         emit_store_dst(jd, iptr, d);
1164                         break;
1165
1166                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1167                                       /* sx.val.l = constant                             */
1168
1169                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1170                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1171                         M_INTMOVE(s1, d);
1172                         if (IS_IMM32(iptr->sx.val.l))
1173                                 M_LXOR_IMM(iptr->sx.val.l, d);
1174                         else {
1175                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1176                                 M_LXOR(REG_ITMP2, d);
1177                         }
1178                         emit_store_dst(jd, iptr, d);
1179                         break;
1180
1181
1182                 /* floating operations ************************************************/
1183
1184                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1185
1186                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1187                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1188                         disp = dseg_add_s4(cd, 0x80000000);
1189                         M_FLTMOVE(s1, d);
1190                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1191                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1192                         emit_store_dst(jd, iptr, d);
1193                         break;
1194
1195                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1196
1197                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1198                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1199                         disp = dseg_add_s8(cd, 0x8000000000000000);
1200                         M_FLTMOVE(s1, d);
1201                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1202                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1203                         emit_store_dst(jd, iptr, d);
1204                         break;
1205
1206                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1207
1208                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1209                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1210                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1211                         if (s2 == d)
1212                                 M_FADD(s1, d);
1213                         else {
1214                                 M_FLTMOVE(s1, d);
1215                                 M_FADD(s2, d);
1216                         }
1217                         emit_store_dst(jd, iptr, d);
1218                         break;
1219
1220                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1221
1222                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1223                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1224                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1225                         if (s2 == d)
1226                                 M_DADD(s1, d);
1227                         else {
1228                                 M_FLTMOVE(s1, d);
1229                                 M_DADD(s2, d);
1230                         }
1231                         emit_store_dst(jd, iptr, d);
1232                         break;
1233
1234                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1235
1236                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1237                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1238                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1239                         if (s2 == d) {
1240                                 M_FLTMOVE(s2, REG_FTMP2);
1241                                 s2 = REG_FTMP2;
1242                         }
1243                         M_FLTMOVE(s1, d);
1244                         M_FSUB(s2, d);
1245                         emit_store_dst(jd, iptr, d);
1246                         break;
1247
1248                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1249
1250                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1251                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1252                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1253                         if (s2 == d) {
1254                                 M_FLTMOVE(s2, REG_FTMP2);
1255                                 s2 = REG_FTMP2;
1256                         }
1257                         M_FLTMOVE(s1, d);
1258                         M_DSUB(s2, d);
1259                         emit_store_dst(jd, iptr, d);
1260                         break;
1261
1262                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1263
1264                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1265                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1266                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1267                         if (s2 == d)
1268                                 M_FMUL(s1, d);
1269                         else {
1270                                 M_FLTMOVE(s1, d);
1271                                 M_FMUL(s2, d);
1272                         }
1273                         emit_store_dst(jd, iptr, d);
1274                         break;
1275
1276                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1277
1278                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1279                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1280                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1281                         if (s2 == d)
1282                                 M_DMUL(s1, d);
1283                         else {
1284                                 M_FLTMOVE(s1, d);
1285                                 M_DMUL(s2, d);
1286                         }
1287                         emit_store_dst(jd, iptr, d);
1288                         break;
1289
1290                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1291
1292                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1293                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1294                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1295                         if (s2 == d) {
1296                                 M_FLTMOVE(s2, REG_FTMP2);
1297                                 s2 = REG_FTMP2;
1298                         }
1299                         M_FLTMOVE(s1, d);
1300                         M_FDIV(s2, d);
1301                         emit_store_dst(jd, iptr, d);
1302                         break;
1303
1304                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1305
1306                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1307                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1308                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1309                         if (s2 == d) {
1310                                 M_FLTMOVE(s2, REG_FTMP2);
1311                                 s2 = REG_FTMP2;
1312                         }
1313                         M_FLTMOVE(s1, d);
1314                         M_DDIV(s2, d);
1315                         emit_store_dst(jd, iptr, d);
1316                         break;
1317
1318                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1319
1320                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1321                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1322                         M_CVTIF(s1, d);
1323                         emit_store_dst(jd, iptr, d);
1324                         break;
1325
1326                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1327
1328                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1329                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1330                         M_CVTID(s1, d);
1331                         emit_store_dst(jd, iptr, d);
1332                         break;
1333
1334                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1335
1336                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1337                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1338                         M_CVTLF(s1, d);
1339                         emit_store_dst(jd, iptr, d);
1340                         break;
1341
1342                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1343
1344                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1345                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1346                         M_CVTLD(s1, d);
1347                         emit_store_dst(jd, iptr, d);
1348                         break;
1349
1350                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1351
1352                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1353                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1354                         M_CVTFI(s1, d);
1355                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1356                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1357                                 ((REG_RESULT == d) ? 0 : 3);
1358                         M_BNE(disp);
1359                         M_FLTMOVE(s1, REG_FTMP1);
1360                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1361                         M_CALL(REG_ITMP2);
1362                         M_INTMOVE(REG_RESULT, d);
1363                         emit_store_dst(jd, iptr, d);
1364                         break;
1365
1366                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1367
1368                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1369                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1370                         M_CVTDI(s1, d);
1371                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1372                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1373                                 ((REG_RESULT == d) ? 0 : 3);
1374                         M_BNE(disp);
1375                         M_FLTMOVE(s1, REG_FTMP1);
1376                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1377                         M_CALL(REG_ITMP2);
1378                         M_INTMOVE(REG_RESULT, d);
1379                         emit_store_dst(jd, iptr, d);
1380                         break;
1381
1382                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1383
1384                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1385                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1386                         M_CVTFL(s1, d);
1387                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1388                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1389                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1390                                 ((REG_RESULT == d) ? 0 : 3);
1391                         M_BNE(disp);
1392                         M_FLTMOVE(s1, REG_FTMP1);
1393                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1394                         M_CALL(REG_ITMP2);
1395                         M_INTMOVE(REG_RESULT, d);
1396                         emit_store_dst(jd, iptr, d);
1397                         break;
1398
1399                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1400
1401                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1402                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1403                         M_CVTDL(s1, d);
1404                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1405                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1406                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1407                                 ((REG_RESULT == d) ? 0 : 3);
1408                         M_BNE(disp);
1409                         M_FLTMOVE(s1, REG_FTMP1);
1410                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1411                         M_CALL(REG_ITMP2);
1412                         M_INTMOVE(REG_RESULT, d);
1413                         emit_store_dst(jd, iptr, d);
1414                         break;
1415
1416                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1417
1418                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1419                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1420                         M_CVTFD(s1, d);
1421                         emit_store_dst(jd, iptr, d);
1422                         break;
1423
1424                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1425
1426                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1427                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1428                         M_CVTDF(s1, d);
1429                         emit_store_dst(jd, iptr, d);
1430                         break;
1431
1432                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1433                                           /* == => 0, < => 1, > => -1 */
1434
1435                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1436                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1437                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1438                         M_CLR(d);
1439                         M_MOV_IMM(1, REG_ITMP1);
1440                         M_MOV_IMM(-1, REG_ITMP2);
1441                         emit_ucomiss_reg_reg(cd, s1, s2);
1442                         M_CMOVULT(REG_ITMP1, d);
1443                         M_CMOVUGT(REG_ITMP2, d);
1444                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1445                         emit_store_dst(jd, iptr, d);
1446                         break;
1447
1448                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1449                                           /* == => 0, < => 1, > => -1 */
1450
1451                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1452                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1453                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1454                         M_CLR(d);
1455                         M_MOV_IMM(1, REG_ITMP1);
1456                         M_MOV_IMM(-1, REG_ITMP2);
1457                         emit_ucomiss_reg_reg(cd, s1, s2);
1458                         M_CMOVULT(REG_ITMP1, d);
1459                         M_CMOVUGT(REG_ITMP2, d);
1460                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1461                         emit_store_dst(jd, iptr, d);
1462                         break;
1463
1464                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1465                                           /* == => 0, < => 1, > => -1 */
1466
1467                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1468                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1469                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1470                         M_CLR(d);
1471                         M_MOV_IMM(1, REG_ITMP1);
1472                         M_MOV_IMM(-1, REG_ITMP2);
1473                         emit_ucomisd_reg_reg(cd, s1, s2);
1474                         M_CMOVULT(REG_ITMP1, d);
1475                         M_CMOVUGT(REG_ITMP2, d);
1476                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1477                         emit_store_dst(jd, iptr, d);
1478                         break;
1479
1480                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1481                                           /* == => 0, < => 1, > => -1 */
1482
1483                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1484                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1485                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1486                         M_CLR(d);
1487                         M_MOV_IMM(1, REG_ITMP1);
1488                         M_MOV_IMM(-1, REG_ITMP2);
1489                         emit_ucomisd_reg_reg(cd, s1, s2);
1490                         M_CMOVULT(REG_ITMP1, d);
1491                         M_CMOVUGT(REG_ITMP2, d);
1492                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1493                         emit_store_dst(jd, iptr, d);
1494                         break;
1495
1496
1497                 /* memory operations **************************************************/
1498
1499                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1500
1501                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1502                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1503                         /* implicit null-pointer check */
1504                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1505                         emit_store_dst(jd, iptr, d);
1506                         break;
1507
1508                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1509
1510                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1511                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1512                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1513                         /* implicit null-pointer check */
1514                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1515                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1516                         emit_store_dst(jd, iptr, d);
1517                         break;
1518
1519                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1520
1521                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1522                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1523                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1524                         /* implicit null-pointer check */
1525                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1526                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1527                         emit_store_dst(jd, iptr, d);
1528                         break;
1529
1530                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1531
1532                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1533                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1534                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1535                         /* implicit null-pointer check */
1536                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1537                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1538                         emit_store_dst(jd, iptr, d);
1539                         break;
1540
1541                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1542
1543                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1544                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1545                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1546                         /* implicit null-pointer check */
1547                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1548                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1549                         emit_store_dst(jd, iptr, d);
1550                         break;
1551
1552                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1553
1554                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1555                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1556                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1557                         /* implicit null-pointer check */
1558                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1559                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1560                         emit_store_dst(jd, iptr, d);
1561                         break;
1562
1563                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1564
1565                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1566                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1567                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1568                         /* implicit null-pointer check */
1569                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1570                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1571                         emit_store_dst(jd, iptr, d);
1572                         break;
1573
1574                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1575
1576                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1577                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1578                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1579                         /* implicit null-pointer check */
1580                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1581                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1582                         emit_store_dst(jd, iptr, d);
1583                         break;
1584
1585                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1586
1587                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1588                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1589                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1590                         /* implicit null-pointer check */
1591                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1592                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1593                         emit_store_dst(jd, iptr, d);
1594                         break;
1595
1596
1597                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1598
1599                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1600                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1601                         /* implicit null-pointer check */
1602                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1603                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1604                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1605                         break;
1606
1607                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1608
1609                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1610                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1611                         /* implicit null-pointer check */
1612                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1613                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1614                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1615                         break;
1616
1617                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1618
1619                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1620                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1621                         /* implicit null-pointer check */
1622                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1623                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1624                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1625                         break;
1626
1627                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1628
1629                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1630                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1631                         /* implicit null-pointer check */
1632                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1633                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1634                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
1635                         break;
1636
1637                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1638
1639                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1640                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1641                         /* implicit null-pointer check */
1642                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1643                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1644                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
1645                         break;
1646
1647                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1648
1649                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1650                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1651                         /* implicit null-pointer check */
1652                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1653                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1654                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
1655                         break;
1656
1657                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1658
1659                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1660                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1661                         /* implicit null-pointer check */
1662                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1663                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1664                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
1665                         break;
1666
1667                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1668
1669                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1670                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1671                         /* implicit null-pointer check */
1672                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1673                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1674
1675                         M_MOV(s1, REG_A0);
1676                         M_MOV(s3, REG_A1);
1677                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
1678                         M_CALL(REG_ITMP1);
1679                         emit_exception_check(cd, iptr);
1680
1681                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1682                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1683                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1684                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1685                         break;
1686
1687
1688                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1689
1690                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1691                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1692                         /* implicit null-pointer check */
1693                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1694                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1695                         break;
1696
1697                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1698
1699                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1700                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1701                         /* implicit null-pointer check */
1702                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1703                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray, data[0]), s1, s2, 1);
1704                         break;
1705
1706                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1707
1708                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1709                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1710                         /* implicit null-pointer check */
1711                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1712                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1713                         break;
1714
1715                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1716
1717                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1718                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1719                         /* implicit null-pointer check */
1720                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1721                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray, data[0]), s1, s2, 2);
1722                         break;
1723
1724                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1725
1726                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1727                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1728                         /* implicit null-pointer check */
1729                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1730
1731                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1732                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1733                         }
1734                         else {
1735                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1736                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
1737                         }
1738                         break;
1739
1740                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1741
1742                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1743                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1744                         /* implicit null-pointer check */
1745                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1746                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1747                         break;
1748
1749
1750                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1751
1752                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1753                                 uf        = iptr->sx.s23.s3.uf;
1754                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1755                                 disp      = dseg_add_unique_address(cd, NULL);
1756                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1757
1758                                 /* must be calculated before codegen_add_patch_ref */
1759
1760                                 if (opt_shownops)
1761                                         disp -= PATCHER_CALL_SIZE;
1762
1763 /*                              PROFILE_CYCLE_STOP; */
1764
1765                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1766
1767 /*                              PROFILE_CYCLE_START; */
1768                         }
1769                         else {
1770                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1771                                 fieldtype = fi->type;
1772                                 disp      = dseg_add_address(cd, fi->value);
1773                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1774
1775                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1776                                         PROFILE_CYCLE_STOP;
1777
1778                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1779
1780                                         PROFILE_CYCLE_START;
1781
1782                                         if (opt_shownops)
1783                                                 disp -= PATCHER_CALL_SIZE;
1784                                 }
1785                         }
1786
1787                         /* This approach is much faster than moving the field
1788                            address inline into a register. */
1789
1790                         M_ALD(REG_ITMP1, RIP, disp);
1791
1792                         switch (fieldtype) {
1793                         case TYPE_INT:
1794                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1795                                 M_ILD(d, REG_ITMP1, 0);
1796                                 break;
1797                         case TYPE_LNG:
1798                         case TYPE_ADR:
1799                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1800                                 M_LLD(d, REG_ITMP1, 0);
1801                                 break;
1802                         case TYPE_FLT:
1803                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1804                                 M_FLD(d, REG_ITMP1, 0);
1805                                 break;
1806                         case TYPE_DBL:
1807                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1808                                 M_DLD(d, REG_ITMP1, 0);
1809                                 break;
1810                         }
1811                         emit_store_dst(jd, iptr, d);
1812                         break;
1813
1814                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1815
1816                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1817                                 uf        = iptr->sx.s23.s3.uf;
1818                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1819                                 disp      = dseg_add_unique_address(cd, NULL);
1820                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1821
1822                                 /* must be calculated before codegen_add_patch_ref */
1823
1824                                 if (opt_shownops)
1825                                         disp -= PATCHER_CALL_SIZE;
1826
1827 /*                              PROFILE_CYCLE_STOP; */
1828
1829                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1830
1831 /*                              PROFILE_CYCLE_START; */
1832                         }
1833                         else {
1834                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1835                                 fieldtype = fi->type;
1836                                 disp      = dseg_add_address(cd, fi->value);
1837                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1838
1839                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1840                                         PROFILE_CYCLE_STOP;
1841
1842                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1843
1844                                         PROFILE_CYCLE_START;
1845
1846                                         if (opt_shownops)
1847                                                 disp -= PATCHER_CALL_SIZE;
1848                                 }
1849                         }
1850
1851                         /* This approach is much faster than moving the field
1852                            address inline into a register. */
1853
1854                         M_ALD(REG_ITMP1, RIP, disp);
1855
1856                         switch (fieldtype) {
1857                         case TYPE_INT:
1858                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1859                                 M_IST(s1, REG_ITMP1, 0);
1860                                 break;
1861                         case TYPE_LNG:
1862                         case TYPE_ADR:
1863                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1864                                 M_LST(s1, REG_ITMP1, 0);
1865                                 break;
1866                         case TYPE_FLT:
1867                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1868                                 M_FST(s1, REG_ITMP1, 0);
1869                                 break;
1870                         case TYPE_DBL:
1871                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1872                                 M_DST(s1, REG_ITMP1, 0);
1873                                 break;
1874                         }
1875                         break;
1876
1877                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1878                                           /* val = value (in current instruction)     */
1879                                           /* following NOP)                           */
1880
1881                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1882                                 uf        = iptr->sx.s23.s3.uf;
1883                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1884                                 disp      = dseg_add_unique_address(cd, NULL);
1885                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1886
1887                                 /* must be calculated before codegen_add_patch_ref */
1888
1889                                 if (opt_shownops)
1890                                         disp -= PATCHER_CALL_SIZE;
1891
1892 /*                              PROFILE_CYCLE_STOP; */
1893
1894                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1895
1896 /*                              PROFILE_CYCLE_START; */
1897                         }
1898                         else {
1899                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1900                                 fieldtype = fi->type;
1901                                 disp      = dseg_add_address(cd, fi->value);
1902                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1903
1904                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1905                                         PROFILE_CYCLE_STOP;
1906
1907                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1908
1909                                         PROFILE_CYCLE_START;
1910
1911                                         if (opt_shownops)
1912                                                 disp -= PATCHER_CALL_SIZE;
1913                                 }
1914                         }
1915
1916                         /* This approach is much faster than moving the field
1917                            address inline into a register. */
1918
1919                         M_ALD(REG_ITMP1, RIP, disp);
1920
1921                         switch (fieldtype) {
1922                         case TYPE_INT:
1923                         case TYPE_FLT:
1924                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1925                                 break;
1926                         case TYPE_LNG:
1927                         case TYPE_ADR:
1928                         case TYPE_DBL:
1929                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1930                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1931                                 else {
1932                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1933                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1934                                 }
1935                                 break;
1936                         }
1937                         break;
1938
1939                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1940
1941                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1942
1943                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1944                                 uf        = iptr->sx.s23.s3.uf;
1945                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1946                                 disp      = 0;
1947
1948 /*                              PROFILE_CYCLE_STOP; */
1949
1950                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1951
1952 /*                              PROFILE_CYCLE_START; */
1953                         }
1954                         else {
1955                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1956                                 fieldtype = fi->type;
1957                                 disp      = fi->offset;
1958                         }
1959
1960                         /* implicit null-pointer check */
1961                         switch (fieldtype) {
1962                         case TYPE_INT:
1963                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1964                                 M_ILD32(d, s1, disp);
1965                                 break;
1966                         case TYPE_LNG:
1967                         case TYPE_ADR:
1968                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1969                                 M_LLD32(d, s1, disp);
1970                                 break;
1971                         case TYPE_FLT:
1972                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1973                                 M_FLD32(d, s1, disp);
1974                                 break;
1975                         case TYPE_DBL:
1976                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1977                                 M_DLD32(d, s1, disp);
1978                                 break;
1979                         }
1980                         emit_store_dst(jd, iptr, d);
1981                         break;
1982
1983                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1984
1985                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1986                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1987
1988                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1989                                 uf        = iptr->sx.s23.s3.uf;
1990                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1991                                 disp      = 0;
1992
1993 /*                              PROFILE_CYCLE_STOP; */
1994
1995                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1996
1997 /*                              PROFILE_CYCLE_START; */
1998                         }
1999                         else {
2000                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2001                                 fieldtype = fi->type;
2002                                 disp      = fi->offset;
2003                         }
2004
2005                         /* implicit null-pointer check */
2006                         switch (fieldtype) {
2007                         case TYPE_INT:
2008                                 M_IST32(s2, s1, disp);
2009                                 break;
2010                         case TYPE_LNG:
2011                         case TYPE_ADR:
2012                                 M_LST32(s2, s1, disp);
2013                                 break;
2014                         case TYPE_FLT:
2015                                 M_FST32(s2, s1, disp);
2016                                 break;
2017                         case TYPE_DBL:
2018                                 M_DST32(s2, s1, disp);
2019                                 break;
2020                         }
2021                         break;
2022
2023                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2024                                           /* val = value (in current instruction)     */
2025                                           /* following NOP)                           */
2026
2027                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2028
2029                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2030                                 uf        = iptr->sx.s23.s3.uf;
2031                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2032                                 disp      = 0;
2033
2034 /*                              PROFILE_CYCLE_STOP; */
2035
2036                                 codegen_add_patch_ref(cd, PATCHER_putfieldconst, uf, 0);
2037
2038 /*                              PROFILE_CYCLE_START; */
2039                         }
2040                         else {
2041                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2042                                 fieldtype = fi->type;
2043                                 disp      = fi->offset;
2044                         }
2045
2046                         /* implicit null-pointer check */
2047                         switch (fieldtype) {
2048                         case TYPE_INT:
2049                         case TYPE_FLT:
2050                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2051                                 break;
2052                         case TYPE_LNG:
2053                         case TYPE_ADR:
2054                         case TYPE_DBL:
2055                                 /* XXX why no check for IS_IMM32? */
2056                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2057                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2058                                 break;
2059                         }
2060                         break;
2061
2062
2063                 /* branch operations **************************************************/
2064
2065                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2066
2067                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2068                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2069
2070                         PROFILE_CYCLE_STOP;
2071
2072 #ifdef ENABLE_VERIFIER
2073                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2074                                 uc = iptr->sx.s23.s2.uc;
2075
2076                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2077                         }
2078 #endif /* ENABLE_VERIFIER */
2079
2080                         M_CALL_IMM(0);                            /* passing exception pc */
2081                         M_POP(REG_ITMP2_XPC);
2082
2083                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2084                         M_JMP(REG_ITMP3);
2085                         break;
2086
2087                 case ICMD_GOTO:         /* ... ==> ...                                */
2088                 case ICMD_RET:
2089
2090                         emit_br(cd, iptr->dst.block);
2091                         ALIGNCODENOP;
2092                         break;
2093
2094                 case ICMD_JSR:          /* ... ==> ...                                */
2095
2096                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2097                         ALIGNCODENOP;
2098                         break;
2099
2100                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2101                 case ICMD_IFNONNULL:
2102
2103                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2104                         M_TEST(s1);
2105                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2106                         break;
2107
2108                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2109                 case ICMD_IFLT:
2110                 case ICMD_IFLE:
2111                 case ICMD_IFNE:
2112                 case ICMD_IFGT:
2113                 case ICMD_IFGE:
2114
2115                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2116                         M_ICMP_IMM(iptr->sx.val.i, s1);
2117                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2118                         break;
2119
2120                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2121                 case ICMD_IF_LNE:
2122                 case ICMD_IF_LLT:
2123                 case ICMD_IF_LGE:
2124                 case ICMD_IF_LGT:
2125                 case ICMD_IF_LLE:
2126
2127                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2128                         if (IS_IMM32(iptr->sx.val.l))
2129                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2130                         else {
2131                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2132                                 M_LCMP(REG_ITMP2, s1);
2133                         }
2134                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2135                         break;
2136
2137                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2138                 case ICMD_IF_ICMPNE:
2139                 case ICMD_IF_ICMPLT:
2140                 case ICMD_IF_ICMPGE:
2141                 case ICMD_IF_ICMPGT:
2142                 case ICMD_IF_ICMPLE:
2143
2144                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2145                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2146                         M_ICMP(s2, s1);
2147                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2148                         break;
2149
2150                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2151                 case ICMD_IF_ACMPNE:
2152
2153                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2154                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2155                         M_LCMP(s2, s1);
2156                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2157                         break;
2158
2159                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2160                 case ICMD_IF_LCMPNE:
2161                 case ICMD_IF_LCMPLT:
2162                 case ICMD_IF_LCMPGE:
2163                 case ICMD_IF_LCMPGT:
2164                 case ICMD_IF_LCMPLE:
2165
2166                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2167                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2168                         M_LCMP(s2, s1);
2169                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2170                         break;
2171
2172                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2173                 case ICMD_LRETURN:
2174
2175                         REPLACEMENT_POINT_RETURN(cd, iptr);
2176                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2177                         M_INTMOVE(s1, REG_RESULT);
2178                         goto nowperformreturn;
2179
2180                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2181
2182                         REPLACEMENT_POINT_RETURN(cd, iptr);
2183                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2184                         M_INTMOVE(s1, REG_RESULT);
2185
2186 #ifdef ENABLE_VERIFIER
2187                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2188                                 uc = iptr->sx.s23.s2.uc;
2189
2190                                 PROFILE_CYCLE_STOP;
2191
2192                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2193
2194                                 PROFILE_CYCLE_START;
2195                         }
2196 #endif /* ENABLE_VERIFIER */
2197                         goto nowperformreturn;
2198
2199                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2200                 case ICMD_DRETURN:
2201
2202                         REPLACEMENT_POINT_RETURN(cd, iptr);
2203                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2204                         M_FLTMOVE(s1, REG_FRESULT);
2205                         goto nowperformreturn;
2206
2207                 case ICMD_RETURN:      /* ...  ==> ...                                */
2208
2209                         REPLACEMENT_POINT_RETURN(cd, iptr);
2210
2211 nowperformreturn:
2212                         {
2213                         s4 i, p;
2214
2215                         p = cd->stackframesize;
2216
2217 #if !defined(NDEBUG)
2218                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2219                                 emit_verbosecall_exit(jd);
2220 #endif /* !defined(NDEBUG) */
2221
2222 #if defined(ENABLE_THREADS)
2223                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2224                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2225
2226                                 /* we need to save the proper return value */
2227                                 switch (iptr->opc) {
2228                                 case ICMD_IRETURN:
2229                                 case ICMD_ARETURN:
2230                                 case ICMD_LRETURN:
2231                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2232                                         break;
2233                                 case ICMD_FRETURN:
2234                                 case ICMD_DRETURN:
2235                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2236                                         break;
2237                                 }
2238
2239                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2240                                 M_CALL(REG_ITMP1);
2241
2242                                 /* and now restore the proper return value */
2243                                 switch (iptr->opc) {
2244                                 case ICMD_IRETURN:
2245                                 case ICMD_ARETURN:
2246                                 case ICMD_LRETURN:
2247                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2248                                         break;
2249                                 case ICMD_FRETURN:
2250                                 case ICMD_DRETURN:
2251                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2252                                         break;
2253                                 }
2254                         }
2255 #endif
2256
2257                         /* restore saved registers */
2258
2259                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2260                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2261                         }
2262                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2263                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2264                         }
2265
2266                         /* deallocate stack */
2267
2268                         if (cd->stackframesize)
2269                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2270
2271                         /* generate method profiling code */
2272
2273                         PROFILE_CYCLE_STOP;
2274
2275                         M_RET;
2276                         }
2277                         break;
2278
2279
2280                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2281                         {
2282                                 s4 i, l;
2283                                 branch_target_t *table;
2284
2285                                 table = iptr->dst.table;
2286
2287                                 l = iptr->sx.s23.s2.tablelow;
2288                                 i = iptr->sx.s23.s3.tablehigh;
2289
2290                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2291                                 M_INTMOVE(s1, REG_ITMP1);
2292
2293                                 if (l != 0)
2294                                         M_ISUB_IMM(l, REG_ITMP1);
2295
2296                                 /* number of targets */
2297                                 i = i - l + 1;
2298
2299                 /* range check */
2300
2301                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2302                                 emit_bugt(cd, table[0].block);
2303
2304                                 /* build jump table top down and use address of lowest entry */
2305
2306                                 table += i;
2307
2308                                 while (--i >= 0) {
2309                                         dseg_add_target(cd, table->block);
2310                                         --table;
2311                                 }
2312
2313                                 /* length of dataseg after last dseg_add_target is used
2314                                    by load */
2315
2316                                 M_MOV_IMM(0, REG_ITMP2);
2317                                 dseg_adddata(cd);
2318                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2319                                 M_JMP(REG_ITMP1);
2320                         }
2321                         break;
2322
2323
2324                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2325                         {
2326                                 s4 i;
2327                                 lookup_target_t *lookup;
2328
2329                                 lookup = iptr->dst.lookup;
2330
2331                                 i = iptr->sx.s23.s2.lookupcount;
2332
2333                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2334                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2335
2336                                 while (--i >= 0) {
2337                                         M_ICMP_IMM(lookup->value, s1);
2338                                         emit_beq(cd, lookup->target.block);
2339                                         lookup++;
2340                                 }
2341
2342                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2343                                 ALIGNCODENOP;
2344                         }
2345                         break;
2346
2347
2348                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2349
2350                         REPLACEMENT_POINT_FORGC_BUILTIN(cd, iptr);
2351
2352                         bte = iptr->sx.s23.s3.bte;
2353                         md  = bte->md;
2354                         goto gen_method;
2355
2356                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2357
2358                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2359                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2360                 case ICMD_INVOKEINTERFACE:
2361
2362                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2363
2364                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2365                                 lm = NULL;
2366                                 um = iptr->sx.s23.s3.um;
2367                                 md = um->methodref->parseddesc.md;
2368                         }
2369                         else {
2370                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2371                                 um = NULL;
2372                                 md = lm->parseddesc;
2373                         }
2374
2375 gen_method:
2376                         s3 = md->paramcount;
2377
2378                         MCODECHECK((20 * s3) + 128);
2379
2380                         /* copy arguments to registers or stack location */
2381
2382                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2383                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2384                                 d   = md->params[s3].regoff;
2385
2386                                 /* already preallocated (ARGVAR)? */
2387
2388                                 if (var->flags & PREALLOC)
2389                                         continue;
2390
2391                                 if (IS_INT_LNG_TYPE(var->type)) {
2392                                         if (!md->params[s3].inmemory) {
2393                                                 s1 = emit_load(jd, iptr, var, d);
2394                                                 M_INTMOVE(s1, d);
2395                                         }
2396                                         else {
2397                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2398                                                 M_LST(s1, REG_SP, d);
2399                                         }
2400                                 }
2401                                 else {
2402                                         if (!md->params[s3].inmemory) {
2403                                                 s1 = emit_load(jd, iptr, var, d);
2404                                                 M_FLTMOVE(s1, d);
2405                                         }
2406                                         else {
2407                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2408
2409                                                 if (IS_2_WORD_TYPE(var->type))
2410                                                         M_DST(s1, REG_SP, d);
2411                                                 else
2412                                                         M_FST(s1, REG_SP, d);
2413                                         }
2414                                 }
2415                         }
2416
2417                         /* generate method profiling code */
2418
2419                         PROFILE_CYCLE_STOP;
2420
2421                         switch (iptr->opc) {
2422                         case ICMD_BUILTIN:
2423                                 if (bte->stub == NULL) {
2424                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2425                                 } else {
2426                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2427                                 }
2428                                 M_CALL(REG_ITMP1);
2429
2430                                 emit_exception_check(cd, iptr);
2431                                 break;
2432
2433                         case ICMD_INVOKESPECIAL:
2434                                 emit_nullpointer_check(cd, iptr, REG_A0);
2435                                 /* fall through */
2436
2437                         case ICMD_INVOKESTATIC:
2438                                 if (lm == NULL) {
2439                                         disp = dseg_add_unique_address(cd, NULL);
2440                                         disp = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
2441
2442                                         /* must be calculated before codegen_add_patch_ref */
2443
2444                                         if (opt_shownops)
2445                                                 disp -= PATCHER_CALL_SIZE;
2446
2447                                         codegen_add_patch_ref(cd, PATCHER_invokestatic_special,
2448                                                                                   um, disp);
2449
2450 /*                                      a = 0; */
2451                                 }
2452                                 else {
2453                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2454                                         disp = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
2455
2456 /*                                      a = (ptrint) lm->stubroutine; */
2457                                 }
2458
2459 /*                              M_MOV_IMM(a, REG_ITMP2); */
2460                                 M_ALD(REG_ITMP2, RIP, disp);
2461                                 M_CALL(REG_ITMP2);
2462                                 break;
2463
2464                         case ICMD_INVOKEVIRTUAL:
2465                                 if (lm == NULL) {
2466                                         codegen_add_patch_ref(cd, PATCHER_invokevirtual, um, 0);
2467
2468                                         s1 = 0;
2469                                 }
2470                                 else {
2471                                         s1 = OFFSET(vftbl_t, table[0]) +
2472                                                 sizeof(methodptr) * lm->vftblindex;
2473                                 }
2474
2475                                 /* implicit null-pointer check */
2476                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2477                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2478                                 M_CALL(REG_ITMP3);
2479                                 break;
2480
2481                         case ICMD_INVOKEINTERFACE:
2482                                 if (lm == NULL) {
2483                                         codegen_add_patch_ref(cd, PATCHER_invokeinterface, um, 0);
2484
2485                                         s1 = 0;
2486                                         s2 = 0;
2487                                 }
2488                                 else {
2489                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2490                                                 sizeof(methodptr) * lm->class->index;
2491
2492                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2493                                 }
2494
2495                                 /* implicit null-pointer check */
2496                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2497                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2498                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2499                                 M_CALL(REG_ITMP3);
2500                                 break;
2501                         }
2502
2503                         /* generate method profiling code */
2504
2505                         PROFILE_CYCLE_START;
2506
2507                         /* store size of call code in replacement point */
2508
2509                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2510                         REPLACEMENT_POINT_FORGC_BUILTIN_RETURN(cd, iptr);
2511
2512                         /* store return value */
2513
2514                         switch (md->returntype.type) {
2515                         case TYPE_INT:
2516                         case TYPE_LNG:
2517                         case TYPE_ADR:
2518                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2519                                 M_INTMOVE(REG_RESULT, s1);
2520                                 emit_store_dst(jd, iptr, s1);
2521                                 break;
2522                         case TYPE_FLT:
2523                         case TYPE_DBL:
2524                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2525                                 M_FLTMOVE(REG_FRESULT, s1);
2526                                 emit_store_dst(jd, iptr, s1);
2527                                 break;
2528                         default:
2529                                 /* TYPE_VOID */
2530                                 break;
2531                         }
2532                         break;
2533
2534
2535                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2536
2537                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2538                                 /* object type cast-check */
2539
2540                                 classinfo *super;
2541                                 vftbl_t   *supervftbl;
2542                                 s4         superindex;
2543
2544                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2545                                         super      = NULL;
2546                                         superindex = 0;
2547                                         supervftbl = NULL;
2548                                 }
2549                                 else {
2550                                         super      = iptr->sx.s23.s3.c.cls;
2551                                         superindex = super->index;
2552                                         supervftbl = super->vftbl;
2553                                 }
2554
2555                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2556                                         CODEGEN_CRITICAL_SECTION_NEW;
2557
2558                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2559
2560                                 /* if class is not resolved, check which code to call */
2561
2562                                 if (super == NULL) {
2563                                         M_TEST(s1);
2564                                         emit_label_beq(cd, BRANCH_LABEL_1);
2565
2566                                         codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2567                                                                                   iptr->sx.s23.s3.c.ref, 0);
2568
2569                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2570                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2571                                         emit_label_beq(cd, BRANCH_LABEL_2);
2572                                 }
2573
2574                                 /* interface checkcast code */
2575
2576                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2577                                         if (super != NULL) {
2578                                                 M_TEST(s1);
2579                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2580                                         }
2581
2582                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2583
2584                                         if (super == NULL) {
2585                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_interface,
2586                                                                                           iptr->sx.s23.s3.c.ref,
2587                                                                                           0);
2588                                         }
2589
2590                                         M_ILD32(REG_ITMP3,
2591                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2592                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2593                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2594
2595                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2596                                                         OFFSET(vftbl_t, interfacetable[0]) -
2597                                                         superindex * sizeof(methodptr*));
2598                                         M_TEST(REG_ITMP3);
2599                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2600
2601                                         if (super == NULL)
2602                                                 emit_label_br(cd, BRANCH_LABEL_4);
2603                                         else
2604                                                 emit_label(cd, BRANCH_LABEL_3);
2605                                 }
2606
2607                                 /* class checkcast code */
2608
2609                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2610                                         if (super == NULL) {
2611                                                 emit_label(cd, BRANCH_LABEL_2);
2612                                         }
2613                                         else {
2614                                                 M_TEST(s1);
2615                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2616                                         }
2617
2618                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2619
2620                                         if (super == NULL) {
2621                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_class,
2622                                                                                           iptr->sx.s23.s3.c.ref,
2623                                                                                           0);
2624                                         }
2625
2626                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2627
2628                                         CODEGEN_CRITICAL_SECTION_START;
2629
2630                                         M_ILD32(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2631
2632                                         /*                                      if (s1 != REG_ITMP1) { */
2633                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2634                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2635                                         /*                                                                                              REG_ITMP1); */
2636                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2637                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2638                                         /*                                                                                              REG_ITMP3); */
2639                                         /*  #if defined(ENABLE_THREADS) */
2640                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2641                                         /*  #endif */
2642                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2643
2644                                         /*                                      } else { */
2645
2646                                         M_ILD32(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
2647                                         M_ISUB(REG_ITMP3, REG_ITMP2);
2648                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2649                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2650                                         /*                                      } */
2651
2652                                         CODEGEN_CRITICAL_SECTION_END;
2653
2654                                         M_ICMP(REG_ITMP3, REG_ITMP2);
2655                                         emit_classcast_check(cd, iptr, BRANCH_UGT, REG_ITMP3, s1);
2656
2657                                         if (super != NULL)
2658                                                 emit_label(cd, BRANCH_LABEL_5);
2659                                 }
2660
2661                                 if (super == NULL) {
2662                                         emit_label(cd, BRANCH_LABEL_1);
2663                                         emit_label(cd, BRANCH_LABEL_4);
2664                                 }
2665
2666                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2667                         }
2668                         else {
2669                                 /* array type cast-check */
2670
2671                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2672                                 M_INTMOVE(s1, REG_A0);
2673
2674                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2675                                         codegen_add_patch_ref(cd, PATCHER_builtin_arraycheckcast,
2676                                                                                   iptr->sx.s23.s3.c.ref, 0);
2677                                 }
2678
2679                                 M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2680                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2681                                 M_CALL(REG_ITMP1);
2682
2683                                 /* s1 may have been destroyed over the function call */
2684                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2685                                 M_TEST(REG_RESULT);
2686                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2687
2688                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2689                         }
2690
2691                         M_INTMOVE(s1, d);
2692                         emit_store_dst(jd, iptr, d);
2693                         break;
2694
2695                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2696
2697                         {
2698                         classinfo *super;
2699                         vftbl_t   *supervftbl;
2700                         s4         superindex;
2701
2702                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2703                                 super      = NULL;
2704                                 superindex = 0;
2705                                 supervftbl = NULL;
2706                         }
2707                         else {
2708                                 super      = iptr->sx.s23.s3.c.cls;
2709                                 superindex = super->index;
2710                                 supervftbl = super->vftbl;
2711                         }
2712
2713                         if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2714                                 CODEGEN_CRITICAL_SECTION_NEW;
2715
2716                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2717                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2718
2719                         if (s1 == d) {
2720                                 M_INTMOVE(s1, REG_ITMP1);
2721                                 s1 = REG_ITMP1;
2722                         }
2723
2724                         M_CLR(d);
2725
2726                         /* if class is not resolved, check which code to call */
2727
2728                         if (super == NULL) {
2729                                 M_TEST(s1);
2730                                 emit_label_beq(cd, BRANCH_LABEL_1);
2731
2732                                 codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2733                                                                           iptr->sx.s23.s3.c.ref, 0);
2734
2735                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2736                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2737                                 emit_label_beq(cd, BRANCH_LABEL_2);
2738                         }
2739
2740                         /* interface instanceof code */
2741
2742                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2743                                 if (super != NULL) {
2744                                         M_TEST(s1);
2745                                         emit_label_beq(cd, BRANCH_LABEL_3);
2746                                 }
2747
2748                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2749
2750                                 if (super == NULL) {
2751                                         codegen_add_patch_ref(cd, PATCHER_instanceof_interface,
2752                                                                                   iptr->sx.s23.s3.c.ref, 0);
2753                                 }
2754
2755                                 M_ILD32(REG_ITMP3,
2756                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2757                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2758
2759                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2760
2761                                 M_BLE(a);
2762                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2763                                                 OFFSET(vftbl_t, interfacetable[0]) -
2764                                                 superindex * sizeof(methodptr*));
2765                                 M_TEST(REG_ITMP1);
2766                                 M_SETNE(d);
2767
2768                                 if (super == NULL)
2769                                         emit_label_br(cd, BRANCH_LABEL_4);
2770                                 else
2771                                         emit_label(cd, BRANCH_LABEL_3);
2772                         }
2773
2774                         /* class instanceof code */
2775
2776                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2777                                 if (super == NULL) {
2778                                         emit_label(cd, BRANCH_LABEL_2);
2779                                 }
2780                                 else {
2781                                         M_TEST(s1);
2782                                         emit_label_beq(cd, BRANCH_LABEL_5);
2783                                 }
2784
2785                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2786
2787                                 if (super == NULL) {
2788                                         codegen_add_patch_ref(cd, PATCHER_instanceof_class,
2789                                                                                   iptr->sx.s23.s3.c.ref, 0);
2790                                 }
2791
2792                                 M_MOV_IMM(supervftbl, REG_ITMP2);
2793
2794                                 CODEGEN_CRITICAL_SECTION_START;
2795
2796                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
2797                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2798                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2799
2800                                 CODEGEN_CRITICAL_SECTION_END;
2801
2802                                 M_ISUB(REG_ITMP2, REG_ITMP1);
2803                                 M_CLR(d); /* may be REG_ITMP2 */
2804                                 M_ICMP(REG_ITMP3, REG_ITMP1);
2805                                 M_SETULE(d);
2806
2807                                 if (super != NULL)
2808                                         emit_label(cd, BRANCH_LABEL_5);
2809                         }
2810
2811                         if (super == NULL) {
2812                                 emit_label(cd, BRANCH_LABEL_1);
2813                                 emit_label(cd, BRANCH_LABEL_4);
2814                         }
2815
2816                         emit_store_dst(jd, iptr, d);
2817                         }
2818                         break;
2819
2820                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2821
2822                         /* check for negative sizes and copy sizes to stack if necessary  */
2823
2824                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2825
2826                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2827
2828                                 /* copy SAVEDVAR sizes to stack */
2829                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2830
2831                                 /* Already Preallocated? */
2832                                 if (!(var->flags & PREALLOC)) {
2833                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2834                                         M_LST(s2, REG_SP, s1 * 8);
2835                                 }
2836                         }
2837
2838                         /* is a patcher function set? */
2839
2840                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2841                                 codegen_add_patch_ref(cd, PATCHER_builtin_multianewarray,
2842                                                                           iptr->sx.s23.s3.c.ref, 0);
2843                         }
2844
2845                         /* a0 = dimension count */
2846
2847                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2848
2849                         /* a1 = classinfo */
2850
2851                         M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2852
2853                         /* a2 = pointer to dimensions = stack pointer */
2854
2855                         M_MOV(REG_SP, REG_A2);
2856
2857                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2858                         M_CALL(REG_ITMP1);
2859
2860                         /* check for exception before result assignment */
2861
2862                         emit_exception_check(cd, iptr);
2863
2864                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2865                         M_INTMOVE(REG_RESULT, s1);
2866                         emit_store_dst(jd, iptr, s1);
2867                         break;
2868
2869                 default:
2870                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2871                                                                                    iptr->opc);
2872                         return false;
2873         } /* switch */
2874
2875         } /* for instruction */
2876
2877         MCODECHECK(512); /* XXX require a lower number? */
2878
2879         /* At the end of a basic block we may have to append some nops,
2880            because the patcher stub calling code might be longer than the
2881            actual instruction. So codepatching does not change the
2882            following block unintentionally. */
2883
2884         if (cd->mcodeptr < cd->lastmcodeptr) {
2885                 while (cd->mcodeptr < cd->lastmcodeptr) {
2886                         M_NOP;
2887                 }
2888         }
2889
2890         } /* if (bptr -> flags >= BBREACHED) */
2891         } /* for basic block */
2892
2893         dseg_createlinenumbertable(cd);
2894
2895         /* generate stubs */
2896
2897         emit_patcher_stubs(jd);
2898
2899         /* everything's ok */
2900
2901         return true;
2902 }
2903
2904
2905 /* codegen_emit_stub_compiler **************************************************
2906
2907    Emit a stub routine which calls the compiler.
2908
2909 *******************************************************************************/
2910
2911 void codegen_emit_stub_compiler(jitdata *jd)
2912 {
2913         methodinfo  *m;
2914         codegendata *cd;
2915
2916         /* get required compiler data */
2917
2918         m  = jd->m;
2919         cd = jd->cd;
2920
2921         /* code for the stub */
2922
2923         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
2924         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
2925         M_JMP(REG_ITMP3);
2926 }
2927
2928
2929 /* codegen_emit_stub_builtin ***************************************************
2930
2931    Creates a stub routine which calls a builtin function.
2932
2933 *******************************************************************************/
2934
2935 void codegen_emit_stub_builtin(jitdata *jd, builtintable_entry *bte)
2936 {
2937         codeinfo    *code;
2938         codegendata *cd;
2939         methoddesc  *md;
2940         s4           i, j;
2941         s4           s1, disp;
2942
2943         /* get required compiler data */
2944
2945         code = jd->code;
2946         cd   = jd->cd;
2947
2948         md = bte->md;
2949
2950         /* calculate stack frame size */
2951
2952         cd->stackframesize =
2953                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
2954                 1;                                                    /* return value */
2955
2956         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
2957
2958         /* create method header */
2959
2960         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2961         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2962         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
2963         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2964         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2965         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2966         (void) dseg_addlinenumbertablesize(cd);
2967         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
2968
2969         /* generate stub code */
2970
2971         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2972
2973 #if defined(ENABLE_GC_CACAO)
2974         /* Save callee saved integer registers in stackframeinfo (GC may
2975            need to recover them during a collection). */
2976
2977         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
2978                 OFFSET(stackframeinfo, intregs);
2979
2980         for (i = 0; i < INT_SAV_CNT; i++)
2981                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2982 #endif
2983
2984         /* save integer and float argument registers */
2985
2986         for (i = 0, j = 0; i < md->paramcount; i++) {
2987                 if (!md->params[i].inmemory) {
2988                         s1 = md->params[i].regoff;
2989
2990                         switch (md->paramtypes[i].type) {
2991                         case TYPE_INT:
2992                         case TYPE_LNG:
2993                         case TYPE_ADR:
2994                                 M_LST(s1, REG_SP, j * 8);
2995                                 break;
2996                         case TYPE_FLT:
2997                         case TYPE_DBL:
2998                                 M_DST(s1, REG_SP, j * 8);
2999                                 break;
3000                         }
3001
3002                         j++;
3003                 }
3004         }
3005
3006         /* create dynamic stack info */
3007
3008         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3009         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3010         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3011         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3012         M_MOV_IMM(codegen_stub_builtin_enter, REG_ITMP1);
3013         M_CALL(REG_ITMP1);
3014
3015         /* restore integer and float argument registers */
3016
3017         for (i = 0, j = 0; i < md->paramcount; i++) {
3018                 if (!md->params[i].inmemory) {
3019                         s1 = md->params[i].regoff;
3020
3021                         switch (md->paramtypes[i].type) {
3022                         case TYPE_INT:
3023                         case TYPE_LNG:
3024                         case TYPE_ADR:
3025                                 M_LLD(s1, REG_SP, j * 8);
3026                                 break;
3027                         case TYPE_FLT:
3028                         case TYPE_DBL:
3029                                 M_DLD(s1, REG_SP, j * 8);
3030                                 break;
3031                         }
3032
3033                         j++;
3034                 }
3035         }
3036
3037         /* call the builtin function */
3038
3039         M_MOV_IMM(bte->fp, REG_ITMP3);
3040         M_CALL(REG_ITMP3);
3041
3042         /* save return value */
3043
3044         switch (md->returntype.type) {
3045         case TYPE_INT:
3046         case TYPE_LNG:
3047         case TYPE_ADR:
3048                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3049                 break;
3050         case TYPE_FLT:
3051         case TYPE_DBL:
3052                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3053                 break;
3054         case TYPE_VOID:
3055                 break;
3056         }
3057
3058         /* remove native stackframe info */
3059
3060         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3061         M_MOV_IMM(codegen_stub_builtin_exit, REG_ITMP1);
3062         M_CALL(REG_ITMP1);
3063
3064         /* restore return value */
3065
3066         switch (md->returntype.type) {
3067         case TYPE_INT:
3068         case TYPE_LNG:
3069         case TYPE_ADR:
3070                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3071                 break;
3072         case TYPE_FLT:
3073         case TYPE_DBL:
3074                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3075                 break;
3076         case TYPE_VOID:
3077                 break;
3078         }
3079
3080 #if defined(ENABLE_GC_CACAO)
3081         /* Restore callee saved integer registers from stackframeinfo (GC
3082            might have modified them during a collection). */
3083
3084         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3085                 OFFSET(stackframeinfo, intregs);
3086
3087         for (i = 0; i < INT_SAV_CNT; i++)
3088                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3089 #endif
3090
3091         /* remove stackframe */
3092
3093         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3094         M_RET;
3095 }
3096
3097
3098 /* codegen_emit_stub_native ****************************************************
3099
3100    Emits a stub routine which calls a native method.
3101
3102 *******************************************************************************/
3103
3104 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f)
3105 {
3106         methodinfo  *m;
3107         codeinfo    *code;
3108         codegendata *cd;
3109         methoddesc  *md;
3110         s4           nativeparams;
3111         s4           i, j;
3112         s4           t;
3113         s4           s1, s2, disp;
3114
3115         /* get required compiler data */
3116
3117         m    = jd->m;
3118         code = jd->code;
3119         cd   = jd->cd;
3120
3121         /* initialize variables */
3122
3123         md = m->parseddesc;
3124         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
3125
3126         /* calculate stack frame size */
3127
3128         cd->stackframesize =
3129                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
3130                 sizeof(localref_table) / SIZEOF_VOID_P +
3131                 md->paramcount +
3132                 1 +                       /* functionptr, TODO: store in data segment */
3133                 nmd->memuse;
3134
3135         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
3136
3137         /* create method header */
3138
3139         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
3140         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
3141         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
3142         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
3143         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
3144         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
3145         (void) dseg_addlinenumbertablesize(cd);
3146         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
3147
3148 #if defined(ENABLE_PROFILING)
3149         /* generate native method profiling code */
3150
3151         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3152                 /* count frequency */
3153
3154                 M_MOV_IMM(code, REG_ITMP3);
3155                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3156         }
3157 #endif
3158
3159         /* generate stub code */
3160
3161         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3162
3163 #if !defined(NDEBUG)
3164         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3165                 emit_verbosecall_enter(jd);
3166 #endif
3167
3168         /* get function address (this must happen before the stackframeinfo) */
3169
3170 #if !defined(WITH_STATIC_CLASSPATH)
3171         if (f == NULL)
3172                 codegen_add_patch_ref(cd, PATCHER_resolve_native, m, 0);
3173 #endif
3174
3175         M_MOV_IMM(f, REG_ITMP3);
3176
3177 #if defined(ENABLE_GC_CACAO)
3178         /* Save callee saved integer registers in stackframeinfo (GC may
3179            need to recover them during a collection). */
3180
3181         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3182                 OFFSET(stackframeinfo, intregs);
3183
3184         for (i = 0; i < INT_SAV_CNT; i++)
3185                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3186 #endif
3187
3188         /* save integer and float argument registers */
3189
3190         for (i = 0; i < md->paramcount; i++) {
3191                 if (!md->params[i].inmemory) {
3192                         s1 = md->params[i].regoff;
3193
3194                         switch (md->paramtypes[i].type) {
3195                         case TYPE_INT:
3196                         case TYPE_LNG:
3197                         case TYPE_ADR:
3198                                 M_LST(s1, REG_SP, i * 8);
3199                                 break;
3200                         case TYPE_FLT:
3201                         case TYPE_DBL:
3202                                 M_DST(s1, REG_SP, i * 8);
3203                                 break;
3204                         }
3205                 }
3206         }
3207
3208         M_AST(REG_ITMP3, REG_SP, md->paramcount * 8);
3209
3210         /* create dynamic stack info */
3211
3212         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3213         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3214         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3215         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3216         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3217         M_CALL(REG_ITMP1);
3218
3219         /* restore integer and float argument registers */
3220
3221         for (i = 0; i < md->paramcount; i++) {
3222                 if (!md->params[i].inmemory) {
3223                         s1 = md->params[i].regoff;
3224
3225                         switch (md->paramtypes[i].type) {
3226                         case TYPE_INT:
3227                         case TYPE_LNG:
3228                         case TYPE_ADR:
3229                                 M_LLD(s1, REG_SP, i * 8);
3230                                 break;
3231                         case TYPE_FLT:
3232                         case TYPE_DBL:
3233                                 M_DLD(s1, REG_SP, i * 8);
3234                                 break;
3235                         }
3236                 }
3237         }
3238
3239         M_ALD(REG_ITMP3, REG_SP, md->paramcount * 8);
3240
3241         /* copy or spill arguments to new locations */
3242
3243         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3244                 t  = md->paramtypes[i].type;
3245                 s2 = nmd->params[j].regoff;
3246
3247                 if (IS_INT_LNG_TYPE(t)) {
3248                         if (!md->params[i].inmemory) {
3249                                 s1 = md->params[i].regoff;
3250
3251                                 if (!nmd->params[j].inmemory)
3252                                         M_INTMOVE(s1, s2);
3253                                 else
3254                                         M_LST(s1, REG_SP, s2);
3255                         }
3256                         else {
3257                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3258                                 M_LLD(REG_ITMP1, REG_SP, s1);
3259                                 M_LST(REG_ITMP1, REG_SP, s2);
3260                         }
3261                 }
3262                 else {
3263                         /* We only copy spilled float arguments, as the float
3264                            argument registers keep unchanged. */
3265
3266                         if (md->params[i].inmemory) {
3267                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3268
3269                                 if (IS_2_WORD_TYPE(t)) {
3270                                         M_DLD(REG_FTMP1, REG_SP, s1);
3271                                         M_DST(REG_FTMP1, REG_SP, s2);
3272                                 }
3273                                 else {
3274                                         M_FLD(REG_FTMP1, REG_SP, s1);
3275                                         M_FST(REG_FTMP1, REG_SP, s2);
3276                                 }
3277                         }
3278                 }
3279         }
3280
3281         /* put class into second argument register */
3282
3283         if (m->flags & ACC_STATIC)
3284                 M_MOV_IMM(m->class, REG_A1);
3285
3286         /* put env into first argument register */
3287
3288         M_MOV_IMM(_Jv_env, REG_A0);
3289
3290         /* do the native function call */
3291
3292         M_CALL(REG_ITMP3);
3293
3294         /* save return value */
3295
3296         switch (md->returntype.type) {
3297         case TYPE_INT:
3298         case TYPE_LNG:
3299         case TYPE_ADR:
3300                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3301                 break;
3302         case TYPE_FLT:
3303         case TYPE_DBL:
3304                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3305                 break;
3306         case TYPE_VOID:
3307                 break;
3308         }
3309
3310 #if !defined(NDEBUG)
3311         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3312                 emit_verbosecall_exit(jd);
3313 #endif
3314
3315         /* remove native stackframe info */
3316
3317         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3318         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3319         M_CALL(REG_ITMP1);
3320         M_MOV(REG_RESULT, REG_ITMP3);
3321
3322         /* restore return value */
3323
3324         switch (md->returntype.type) {
3325         case TYPE_INT:
3326         case TYPE_LNG:
3327         case TYPE_ADR:
3328                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3329                 break;
3330         case TYPE_FLT:
3331         case TYPE_DBL:
3332                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3333                 break;
3334         case TYPE_VOID:
3335                 break;
3336         }
3337
3338 #if defined(ENABLE_GC_CACAO)
3339         /* Restore callee saved integer registers from stackframeinfo (GC
3340            might have modified them during a collection). */
3341
3342         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3343                 OFFSET(stackframeinfo, intregs);
3344
3345         for (i = 0; i < INT_SAV_CNT; i++)
3346                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3347 #endif
3348
3349         /* remove stackframe */
3350
3351         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3352
3353         /* test for exception */
3354
3355         M_TEST(REG_ITMP3);
3356         M_BNE(1);
3357         M_RET;
3358
3359         /* handle exception */
3360
3361         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3362         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3363         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3364
3365         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3366         M_JMP(REG_ITMP3);
3367
3368         /* generate patcher stubs */
3369
3370         emit_patcher_stubs(jd);
3371 }
3372
3373
3374 /*
3375  * These are local overrides for various environment variables in Emacs.
3376  * Please do not remove this and leave it at the end of the file, where
3377  * Emacs will automagically detect them.
3378  * ---------------------------------------------------------------------
3379  * Local variables:
3380  * mode: c
3381  * indent-tabs-mode: t
3382  * c-basic-offset: 4
3383  * tab-width: 4
3384  * End:
3385  * vim:noexpandtab:sw=4:ts=4:
3386  */