src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    $Id: codegen.c 7794 2007-04-23 19:57:45Z michi $
  26
  27 */
  28
  29
  30 #include "config.h"
  31
  32 #include <assert.h>
  33 #include <stdio.h>
  34
  35 #include "vm/types.h"
  36
  37 #include "md-abi.h"
  38
  39 #include "vm/jit/x86_64/arch.h"
  40 #include "vm/jit/x86_64/codegen.h"
  41 #include "vm/jit/x86_64/emit.h"
  42
  43 #include "mm/memory.h"
  44 #include "native/jni.h"
  45 #include "native/native.h"
  46
  47 #if defined(ENABLE_THREADS)
  48 # include "threads/native/lock.h"
  49 #endif
  50
  51 #include "vm/builtin.h"
  52 #include "vm/exceptions.h"
  53 #include "vm/global.h"
  54 #include "vm/stringlocal.h"
  55 #include "vm/vm.h"
  56
  57 #include "vm/jit/abi.h"
  58 #include "vm/jit/asmpart.h"
  59 #include "vm/jit/codegen-common.h"
  60 #include "vm/jit/dseg.h"
  61 #include "vm/jit/emit-common.h"
  62 #include "vm/jit/jit.h"
  63 #include "vm/jit/methodheader.h"
  64 #include "vm/jit/parse.h"
  65 #include "vm/jit/patcher.h"
  66 #include "vm/jit/reg.h"
  67 #include "vm/jit/replace.h"
  68 #include "vm/jit/stacktrace.h"
  69
  70 #if defined(ENABLE_LSRA)
  71 # include "vm/jit/allocator/lsra.h"
  72 #endif
  73
  74 #include "vmcore/loader.h"
  75 #include "vmcore/options.h"
  76 #include "vmcore/statistics.h"
  77
  78
  79 /* codegen_emit ****************************************************************
  80
  81    Generates machine code.
  82
  83 *******************************************************************************/
  84
  85 bool codegen_emit(jitdata *jd)
  86 {
  87         methodinfo         *m;
  88         codeinfo           *code;
  89         codegendata        *cd;
  90         registerdata       *rd;
  91         s4                  len, s1, s2, s3, d, disp;
  92         u2                  currentline;
  93         ptrint              a;
  94         varinfo            *var, *dst;
  95         basicblock         *bptr;
  96         instruction        *iptr;
  97         exception_entry    *ex;
  98         constant_classref  *cr;
  99         unresolved_class   *uc;
 100         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 101         unresolved_method  *um;
 102         builtintable_entry *bte;
 103         methoddesc         *md;
 104         fieldinfo          *fi;
 105         unresolved_field   *uf;
 106         s4                  fieldtype;
 107         s4                 varindex;
 108
 109         /* get required compiler data */
 110
 111         m    = jd->m;
 112         code = jd->code;
 113         cd   = jd->cd;
 114         rd   = jd->rd;
 115
 116         /* prevent compiler warnings */
 117
 118         d   = 0;
 119         lm  = NULL;
 120         um  = NULL;
 121         bte = NULL;
 122
 123         {
 124         s4 i, p, t, l;
 125         s4 savedregs_num;
 126
 127         savedregs_num = 0;
 128
 129         /* space to save used callee saved registers */
 130
 131         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 132         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 133
 134         cd->stackframesize = rd->memuse + savedregs_num;
 135
 136 #if defined(ENABLE_THREADS)
 137         /* space to save argument of monitor_enter */
 138
 139         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 140                 cd->stackframesize++;
 141 #endif
 142
 143         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 144            native code e.g. libc or jni (alignment problems with
 145            movaps). */
 146
 147         if (!jd->isleafmethod || opt_verbosecall)
 148                 cd->stackframesize |= 0x1;
 149
 150         /* create method header */
 151
 152         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 153         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 154
 155 #if defined(ENABLE_THREADS)
 156         /* IsSync contains the offset relative to the stack pointer for the
 157            argument of monitor_exit used in the exception handler. Since the
 158            offset could be zero and give a wrong meaning of the flag it is
 159            offset by one.
 160         */
 161
 162         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 163                 (void) dseg_add_unique_s4(cd, (rd->memuse + 1) * 8); /* IsSync        */
 164         else
 165 #endif
 166                 (void) dseg_add_unique_s4(cd, 0);                  /* IsSync          */
 167
 168         (void) dseg_add_unique_s4(cd, jd->isleafmethod);               /* IsLeaf  */
 169         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 170         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 171
 172         (void) dseg_addlinenumbertablesize(cd);
 173
 174         (void) dseg_add_unique_s4(cd, jd->exceptiontablelength); /* ExTableSize   */
 175
 176         /* create exception table */
 177
 178         for (ex = jd->exceptiontable; ex != NULL; ex = ex->down) {
 179                 dseg_add_target(cd, ex->start);
 180                 dseg_add_target(cd, ex->end);
 181                 dseg_add_target(cd, ex->handler);
 182                 (void) dseg_add_unique_address(cd, ex->catchtype.any);
 183         }
 184
 185 #if defined(ENABLE_PROFILING)
 186         /* generate method profiling code */
 187
 188         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 189                 /* count frequency */
 190
 191                 M_MOV_IMM(code, REG_ITMP3);
 192                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 193
 194                 PROFILE_CYCLE_START;
 195         }
 196 #endif
 197
 198         /* create stack frame (if necessary) */
 199
 200         if (cd->stackframesize)
 201                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 202
 203         /* save used callee saved registers */
 204
 205         p = cd->stackframesize;
 206         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 207                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 208         }
 209         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 210                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 211         }
 212
 213         /* take arguments out of register or stack frame */
 214
 215         md = m->parseddesc;
 216
 217         for (p = 0, l = 0; p < md->paramcount; p++) {
 218                 t = md->paramtypes[p].type;
 219
 220                 varindex = jd->local_map[l * 5 + t];
 221
 222                 l++;
 223                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 224                         l++;
 225
 226                 if (varindex == UNUSED)
 227                         continue;
 228
 229                 var = VAR(varindex);
 230
 231                 s1 = md->params[p].regoff;
 232
 233                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 234                         if (!md->params[p].inmemory) {           /* register arguments    */
 235                                 if (!IS_INMEMORY(var->flags))
 236                                         M_INTMOVE(s1, var->vv.regoff);
 237                                 else
 238                                     M_LST(s1, REG_SP, var->vv.regoff * 8);
 239                         }
 240                         else {                                 /* stack arguments       */
 241                                 if (!IS_INMEMORY(var->flags))
 242                                         /* + 8 for return address */
 243                                         M_LLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 244                                 else
 245                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 246                         }
 247                 }
 248                 else {                                     /* floating args         */
 249                         if (!md->params[p].inmemory) {           /* register arguments    */
 250                                 if (!IS_INMEMORY(var->flags))
 251                                         M_FLTMOVE(s1, var->vv.regoff);
 252                                 else
 253                                         M_DST(s1, REG_SP, var->vv.regoff * 8);
 254                         }
 255                         else {                                 /* stack arguments       */
 256                                 if (!IS_INMEMORY(var->flags))
 257                                         M_DLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 258                                 else
 259                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 260                         }
 261                 }
 262         }
 263
 264         /* save monitorenter argument */
 265
 266 #if defined(ENABLE_THREADS)
 267         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 268                 /* stack offset for monitor argument */
 269
 270                 s1 = rd->memuse;
 271
 272                 if (opt_verbosecall) {
 273                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 274
 275                         for (p = 0; p < INT_ARG_CNT; p++)
 276                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 277
 278                         for (p = 0; p < FLT_ARG_CNT; p++)
 279                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 280
 281                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 282                 }
 283
 284                 /* decide which monitor enter function to call */
 285
 286                 if (m->flags & ACC_STATIC) {
 287                         M_MOV_IMM(&m->class->object.header, REG_A0);
 288                 }
 289                 else {
 290                         M_TEST(REG_A0);
 291                         M_BNE(8);
 292                         M_ALD_MEM(REG_A0, EXCEPTION_HARDWARE_NULLPOINTER);
 293                 }
 294
 295                 M_AST(REG_A0, REG_SP, s1 * 8);
 296                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 297                 M_CALL(REG_ITMP1);
 298
 299                 if (opt_verbosecall) {
 300                         for (p = 0; p < INT_ARG_CNT; p++)
 301                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 302
 303                         for (p = 0; p < FLT_ARG_CNT; p++)
 304                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 305
 306                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 307                 }
 308         }
 309 #endif
 310
 311 #if !defined(NDEBUG)
 312         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 313                 emit_verbosecall_enter(jd);
 314 #endif /* !defined(NDEBUG) */
 315
 316         }
 317
 318         /* end of header generation */
 319
 320         /* create replacement points */
 321
 322         REPLACEMENT_POINTS_INIT(cd, jd);
 323
 324         /* walk through all basic blocks */
 325
 326         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 327
 328                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 329
 330                 if (bptr->flags >= BBREACHED) {
 331
 332                 /* branch resolving */
 333
 334                 codegen_resolve_branchrefs(cd, bptr);
 335
 336                 /* handle replacement points */
 337
 338                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 339
 340                 /* copy interface registers to their destination */
 341
 342                 len = bptr->indepth;
 343                 MCODECHECK(512);
 344
 345 #if defined(ENABLE_PROFILING)
 346                 /* generate basicblock profiling code */
 347
 348                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 349                         /* count frequency */
 350
 351                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 352                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 353
 354                         /* if this is an exception handler, start profiling again */
 355
 356                         if (bptr->type == BBTYPE_EXH)
 357                                 PROFILE_CYCLE_START;
 358                 }
 359 #endif
 360
 361 #if defined(ENABLE_LSRA)
 362                 if (opt_lsra) {
 363                         while (len) {
 364                                 len--;
 365                                 src = bptr->invars[len];
 366                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 367                                         if (bptr->type == BBTYPE_EXH) {
 368 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 369                                                 if (!IS_INMEMORY(src->flags))
 370                                                         d= src->vv.regoff;
 371                                                 else
 372                                                         d=REG_ITMP1;
 373                                                 M_INTMOVE(REG_ITMP1, d);
 374                                                 emit_store(jd, NULL, src, d);
 375                                         }
 376                                 }
 377                         }
 378
 379                 } else {
 380 #endif
 381
 382                 while (len) {
 383                         len--;
 384                         var = VAR(bptr->invars[len]);
 385                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 386                                 if (bptr->type == BBTYPE_EXH) {
 387                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 388                                         M_INTMOVE(REG_ITMP1, d);
 389                                         emit_store(jd, NULL, var, d);
 390                                 }
 391                         }
 392                         else {
 393                                 assert((var->flags & INOUT));
 394                         }
 395                 }
 396 #if defined(ENABLE_LSRA)
 397                 }
 398 #endif
 399                 /* walk through all instructions */
 400
 401                 len = bptr->icount;
 402                 currentline = 0;
 403
 404                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 405                         if (iptr->line != currentline) {
 406                                 dseg_addlinenumber(cd, iptr->line);
 407                                 currentline = iptr->line;
 408                         }
 409
 410                         MCODECHECK(1024);                         /* 1KB should be enough */
 411
 412                 switch (iptr->opc) {
 413                 case ICMD_NOP:        /* ...  ==> ...                                 */
 414                 case ICMD_POP:        /* ..., value  ==> ...                          */
 415                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 416                         break;
 417
 418                 case ICMD_INLINE_START:
 419
 420                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 421                         break;
 422
 423                 case ICMD_INLINE_BODY:
 424
 425                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 426                         dseg_addlinenumber_inline_start(cd, iptr);
 427                         dseg_addlinenumber(cd, iptr->line);
 428                         break;
 429
 430                 case ICMD_INLINE_END:
 431
 432                         dseg_addlinenumber_inline_end(cd, iptr);
 433                         dseg_addlinenumber(cd, iptr->line);
 434                         break;
 435
 436                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 437
 438                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 439                         emit_nullpointer_check(cd, iptr, s1);
 440                         break;
 441
 442                 /* constant operations ************************************************/
 443
 444                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 445
 446                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 447                         ICONST(d, iptr->sx.val.i);
 448                         emit_store_dst(jd, iptr, d);
 449                         break;
 450
 451                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 452
 453                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 454                         LCONST(d, iptr->sx.val.l);
 455                         emit_store_dst(jd, iptr, d);
 456                         break;
 457
 458                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 459
 460                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 461                         disp = dseg_add_float(cd, iptr->sx.val.f);
 462                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 463                         emit_store_dst(jd, iptr, d);
 464                         break;
 465
 466                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 467
 468                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 469                         disp = dseg_add_double(cd, iptr->sx.val.d);
 470                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 471                         emit_store_dst(jd, iptr, d);
 472                         break;
 473
 474                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 475
 476                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 477
 478                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 479                                 cr = iptr->sx.val.c.ref;
 480
 481 /*                              PROFILE_CYCLE_STOP; */
 482
 483                                 codegen_add_patch_ref(cd, PATCHER_aconst, cr, 0);
 484
 485 /*                              PROFILE_CYCLE_START; */
 486
 487                                 M_MOV_IMM(NULL, d);
 488
 489                         } else {
 490                                 if (iptr->sx.val.anyptr == 0)
 491                                         M_CLR(d);
 492                                 else
 493                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 494                         }
 495                         emit_store_dst(jd, iptr, d);
 496                         break;
 497
 498
 499                 /* load/store/copy/move operations ************************************/
 500
 501                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 502                 case ICMD_ALOAD:      /* s1 = local variable                          */
 503                 case ICMD_LLOAD:
 504                 case ICMD_FLOAD:
 505                 case ICMD_DLOAD:
 506                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 507                 case ICMD_LSTORE:
 508                 case ICMD_FSTORE:
 509                 case ICMD_DSTORE:
 510                 case ICMD_COPY:
 511                 case ICMD_MOVE:
 512
 513                         emit_copy(jd, iptr);
 514                         break;
 515
 516                 case ICMD_ASTORE:
 517                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 518                                 emit_copy(jd, iptr);
 519                         break;
 520
 521                 /* integer operations *************************************************/
 522
 523                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 524
 525                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 526                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 527                         M_INTMOVE(s1, d);
 528                         M_INEG(d);
 529                         emit_store_dst(jd, iptr, d);
 530                         break;
 531
 532                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 533
 534                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 535                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 536                         M_INTMOVE(s1, d);
 537                         M_LNEG(d);
 538                         emit_store_dst(jd, iptr, d);
 539                         break;
 540
 541                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 542
 543                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 544                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 545                         M_ISEXT(s1, d);
 546                         emit_store_dst(jd, iptr, d);
 547                         break;
 548
 549                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 550
 551                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 552                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 553                         M_IMOV(s1, d);
 554                         emit_store_dst(jd, iptr, d);
 555                         break;
 556
 557                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 558
 559                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 560                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 561                         M_BSEXT(s1, d);
 562                         emit_store_dst(jd, iptr, d);
 563                         break;
 564
 565                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 566
 567                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 568                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 569                         M_CZEXT(s1, d);
 570                         emit_store_dst(jd, iptr, d);
 571                         break;
 572
 573                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 574
 575                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 576                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 577                         M_SSEXT(s1, d);
 578                         emit_store_dst(jd, iptr, d);
 579                         break;
 580
 581
 582                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 583
 584                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 585                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 586                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 587                         if (s2 == d)
 588                                 M_IADD(s1, d);
 589                         else {
 590                                 M_INTMOVE(s1, d);
 591                                 M_IADD(s2, d);
 592                         }
 593                         emit_store_dst(jd, iptr, d);
 594                         break;
 595
 596                 case ICMD_IINC:
 597                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 598                                       /* sx.val.i = constant                             */
 599
 600                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 601                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 602
 603                         /* Using inc and dec is not faster than add (tested with
 604                            sieve). */
 605
 606                         M_INTMOVE(s1, d);
 607                         M_IADD_IMM(iptr->sx.val.i, d);
 608                         emit_store_dst(jd, iptr, d);
 609                         break;
 610
 611                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 612
 613                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 614                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 615                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 616                         if (s2 == d)
 617                                 M_LADD(s1, d);
 618                         else {
 619                                 M_INTMOVE(s1, d);
 620                                 M_LADD(s2, d);
 621                         }
 622                         emit_store_dst(jd, iptr, d);
 623                         break;
 624
 625                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 626                                       /* sx.val.l = constant                             */
 627
 628                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 629                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 630                         M_INTMOVE(s1, d);
 631                         if (IS_IMM32(iptr->sx.val.l))
 632                                 M_LADD_IMM(iptr->sx.val.l, d);
 633                         else {
 634                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 635                                 M_LADD(REG_ITMP2, d);
 636                         }
 637                         emit_store_dst(jd, iptr, d);
 638                         break;
 639
 640                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 641
 642                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 643                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 644                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 645                         if (s2 == d) {
 646                                 M_INTMOVE(s1, REG_ITMP1);
 647                                 M_ISUB(s2, REG_ITMP1);
 648                                 M_INTMOVE(REG_ITMP1, d);
 649                         } else {
 650                                 M_INTMOVE(s1, d);
 651                                 M_ISUB(s2, d);
 652                         }
 653                         emit_store_dst(jd, iptr, d);
 654                         break;
 655
 656                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 657                                       /* sx.val.i = constant                             */
 658
 659                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 660                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 661                         M_INTMOVE(s1, d);
 662                         M_ISUB_IMM(iptr->sx.val.i, d);
 663                         emit_store_dst(jd, iptr, d);
 664                         break;
 665
 666                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 667
 668                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 669                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 670                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 671                         if (s2 == d) {
 672                                 M_INTMOVE(s1, REG_ITMP1);
 673                                 M_LSUB(s2, REG_ITMP1);
 674                                 M_INTMOVE(REG_ITMP1, d);
 675                         } else {
 676                                 M_INTMOVE(s1, d);
 677                                 M_LSUB(s2, d);
 678                         }
 679                         emit_store_dst(jd, iptr, d);
 680                         break;
 681
 682                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 683                                       /* sx.val.l = constant                             */
 684
 685                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 686                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 687                         M_INTMOVE(s1, d);
 688                         if (IS_IMM32(iptr->sx.val.l))
 689                                 M_LSUB_IMM(iptr->sx.val.l, d);
 690                         else {
 691                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 692                                 M_LSUB(REG_ITMP2, d);
 693                         }
 694                         emit_store_dst(jd, iptr, d);
 695                         break;
 696
 697                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 698
 699                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 700                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 701                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 702                         if (s2 == d)
 703                                 M_IMUL(s1, d);
 704                         else {
 705                                 M_INTMOVE(s1, d);
 706                                 M_IMUL(s2, d);
 707                         }
 708                         emit_store_dst(jd, iptr, d);
 709                         break;
 710
 711                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 712                                       /* sx.val.i = constant                             */
 713
 714                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 715                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 716                         if (iptr->sx.val.i == 2) {
 717                                 M_INTMOVE(s1, d);
 718                                 M_ISLL_IMM(1, d);
 719                         } else
 720                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 721                         emit_store_dst(jd, iptr, d);
 722                         break;
 723
 724                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 725
 726                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 727                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 728                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 729                         if (s2 == d)
 730                                 M_LMUL(s1, d);
 731                         else {
 732                                 M_INTMOVE(s1, d);
 733                                 M_LMUL(s2, d);
 734                         }
 735                         emit_store_dst(jd, iptr, d);
 736                         break;
 737
 738                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 739                                       /* sx.val.l = constant                             */
 740
 741                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 742                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 743                         if (IS_IMM32(iptr->sx.val.l))
 744                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 745                         else {
 746                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 747                                 M_INTMOVE(s1, d);
 748                                 M_LMUL(REG_ITMP2, d);
 749                         }
 750                         emit_store_dst(jd, iptr, d);
 751                         break;
 752
 753                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 754
 755                         s1 = emit_load_s1(jd, iptr, RAX);
 756                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 757                         d = codegen_reg_of_dst(jd, iptr, RAX);
 758
 759                         M_INTMOVE(s1, RAX);
 760                         M_INTMOVE(s2, REG_ITMP3);
 761                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 762
 763                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 764
 765                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 766                         M_BNE(4 + 6);
 767                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 768                         M_BEQ(1 + 3);                                          /* 6 bytes */
 769
 770                         emit_cltd(cd);                                         /* 1 byte  */
 771                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 772
 773                         M_INTMOVE(RAX, d);
 774                         emit_store_dst(jd, iptr, d);
 775                         dst = VAROP(iptr->dst);
 776                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 777                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 778                         break;
 779
 780                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 781
 782                         s1 = emit_load_s1(jd, iptr, RAX);
 783                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 784                         d = codegen_reg_of_dst(jd, iptr, RDX);
 785
 786                         M_INTMOVE(s1, RAX);
 787                         M_INTMOVE(s2, REG_ITMP3);
 788                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 789
 790                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 791
 792                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 793                         M_BNE(3 + 4 + 6);
 794                         M_CLR(RDX);                                            /* 3 bytes */
 795                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 796                         M_BEQ(1 + 3);                                          /* 6 bytes */
 797
 798                         emit_cltd(cd);                                         /* 1 byte  */
 799                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 800
 801                         M_INTMOVE(RDX, d);
 802                         emit_store_dst(jd, iptr, d);
 803                         dst = VAROP(iptr->dst);
 804                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 805                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 806                         break;
 807
 808                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 809                                       /* sx.val.i = constant                             */
 810
 811                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 812                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 813                         M_INTMOVE(s1, REG_ITMP1);
 814                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 815                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 816                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 817                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 818                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 819                         emit_store_dst(jd, iptr, d);
 820                         break;
 821
 822                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 823                                       /* sx.val.i = constant                             */
 824
 825                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 826                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 827                         M_INTMOVE(s1, REG_ITMP1);
 828                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 829                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 830                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 831                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 832                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 833                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 834                         emit_store_dst(jd, iptr, d);
 835                         break;
 836
 837
 838                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 839
 840                         s1 = emit_load_s1(jd, iptr, RAX);
 841                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 842                         d = codegen_reg_of_dst(jd, iptr, RAX);
 843
 844                         M_INTMOVE(s1, RAX);
 845                         M_INTMOVE(s2, REG_ITMP3);
 846                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 847
 848                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 849
 850                         /* check as described in jvm spec */
 851                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 852                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 853                         M_BNE(4 + 6);
 854                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 855                         M_BEQ(2 + 3);                                          /* 6 bytes */
 856
 857                         emit_cqto(cd);                                         /* 2 bytes */
 858                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 859
 860                         M_INTMOVE(RAX, d);
 861                         emit_store_dst(jd, iptr, d);
 862                         dst = VAROP(iptr->dst);
 863                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 864                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 865                         break;
 866
 867                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 868
 869                         s1 = emit_load_s1(jd, iptr, RAX);
 870                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 871                         d = codegen_reg_of_dst(jd, iptr, RDX);
 872
 873                         M_INTMOVE(s1, RAX);
 874                         M_INTMOVE(s2, REG_ITMP3);
 875                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 876
 877                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 878
 879                         /* check as described in jvm spec */
 880                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 881                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 882                         M_BNE(3 + 4 + 6);
 883                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 884                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 885                         M_BEQ(2 + 3);                                          /* 6 bytes */
 886
 887                         emit_cqto(cd);                                         /* 2 bytes */
 888                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 889
 890                         M_INTMOVE(RDX, d);
 891                         emit_store_dst(jd, iptr, d);
 892                         dst = VAROP(iptr->dst);
 893                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 894                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 895                         break;
 896
 897                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 898                                       /* sx.val.i = constant                             */
 899
 900                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 901                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 902                         M_INTMOVE(s1, REG_ITMP1);
 903                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 904                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 905                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 906                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 907                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 908                         emit_store_dst(jd, iptr, d);
 909                         break;
 910
 911                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 912                                       /* sx.val.l = constant                             */
 913
 914                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 915                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 916                         M_INTMOVE(s1, REG_ITMP1);
 917                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 918                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 919                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 920                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 921                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 922                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 923                         emit_store_dst(jd, iptr, d);
 924                         break;
 925
 926                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 927
 928                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 929                         emit_ishift(jd, SHIFT_SHL, iptr);
 930                         break;
 931
 932                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 933                                       /* sx.val.i = constant                             */
 934
 935                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 936                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 937                         M_INTMOVE(s1, d);
 938                         M_ISLL_IMM(iptr->sx.val.i, d);
 939                         emit_store_dst(jd, iptr, d);
 940                         break;
 941
 942                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 943
 944                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 945                         emit_ishift(jd, SHIFT_SAR, iptr);
 946                         break;
 947
 948                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 949                                       /* sx.val.i = constant                             */
 950
 951                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 952                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 953                         M_INTMOVE(s1, d);
 954                         M_ISRA_IMM(iptr->sx.val.i, d);
 955                         emit_store_dst(jd, iptr, d);
 956                         break;
 957
 958                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 959
 960                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 961                         emit_ishift(jd, SHIFT_SHR, iptr);
 962                         break;
 963
 964                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 965                                       /* sx.val.i = constant                             */
 966
 967                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 968                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 969                         M_INTMOVE(s1, d);
 970                         M_ISRL_IMM(iptr->sx.val.i, d);
 971                         emit_store_dst(jd, iptr, d);
 972                         break;
 973
 974                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 975
 976                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 977                         emit_lshift(jd, SHIFT_SHL, iptr);
 978                         break;
 979
 980         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 981                                           /* sx.val.i = constant                             */
 982
 983                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 984                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 985                         M_INTMOVE(s1, d);
 986                         M_LSLL_IMM(iptr->sx.val.i, d);
 987                         emit_store_dst(jd, iptr, d);
 988                         break;
 989
 990                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 991
 992                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 993                         emit_lshift(jd, SHIFT_SAR, iptr);
 994                         break;
 995
 996                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 997                                       /* sx.val.i = constant                             */
 998
 999                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1000                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1001                         M_INTMOVE(s1, d);
1002                         M_LSRA_IMM(iptr->sx.val.i, d);
1003                         emit_store_dst(jd, iptr, d);
1004                         break;
1005
1006                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1007
1008                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1009                         emit_lshift(jd, SHIFT_SHR, iptr);
1010                         break;
1011
1012                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1013                                       /* sx.val.l = constant                             */
1014
1015                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1016                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1017                         M_INTMOVE(s1, d);
1018                         M_LSRL_IMM(iptr->sx.val.i, d);
1019                         emit_store_dst(jd, iptr, d);
1020                         break;
1021
1022                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1023
1024                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1025                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1026                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1027                         if (s2 == d)
1028                                 M_IAND(s1, d);
1029                         else {
1030                                 M_INTMOVE(s1, d);
1031                                 M_IAND(s2, d);
1032                         }
1033                         emit_store_dst(jd, iptr, d);
1034                         break;
1035
1036                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1037                                       /* sx.val.i = constant                             */
1038
1039                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1040                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1041                         M_INTMOVE(s1, d);
1042                         M_IAND_IMM(iptr->sx.val.i, d);
1043                         emit_store_dst(jd, iptr, d);
1044                         break;
1045
1046                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1047
1048                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1049                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1050                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1051                         if (s2 == d)
1052                                 M_LAND(s1, d);
1053                         else {
1054                                 M_INTMOVE(s1, d);
1055                                 M_LAND(s2, d);
1056                         }
1057                         emit_store_dst(jd, iptr, d);
1058                         break;
1059
1060                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1061                                       /* sx.val.l = constant                             */
1062
1063                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1064                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1065                         M_INTMOVE(s1, d);
1066                         if (IS_IMM32(iptr->sx.val.l))
1067                                 M_LAND_IMM(iptr->sx.val.l, d);
1068                         else {
1069                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1070                                 M_LAND(REG_ITMP2, d);
1071                         }
1072                         emit_store_dst(jd, iptr, d);
1073                         break;
1074
1075                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1076
1077                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1078                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1079                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1080                         if (s2 == d)
1081                                 M_IOR(s1, d);
1082                         else {
1083                                 M_INTMOVE(s1, d);
1084                                 M_IOR(s2, d);
1085                         }
1086                         emit_store_dst(jd, iptr, d);
1087                         break;
1088
1089                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1090                                       /* sx.val.i = constant                             */
1091
1092                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1093                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1094                         M_INTMOVE(s1, d);
1095                         M_IOR_IMM(iptr->sx.val.i, d);
1096                         emit_store_dst(jd, iptr, d);
1097                         break;
1098
1099                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1100
1101                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1102                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1103                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1104                         if (s2 == d)
1105                                 M_LOR(s1, d);
1106                         else {
1107                                 M_INTMOVE(s1, d);
1108                                 M_LOR(s2, d);
1109                         }
1110                         emit_store_dst(jd, iptr, d);
1111                         break;
1112
1113                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1114                                       /* sx.val.l = constant                             */
1115
1116                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1117                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1118                         M_INTMOVE(s1, d);
1119                         if (IS_IMM32(iptr->sx.val.l))
1120                                 M_LOR_IMM(iptr->sx.val.l, d);
1121                         else {
1122                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1123                                 M_LOR(REG_ITMP2, d);
1124                         }
1125                         emit_store_dst(jd, iptr, d);
1126                         break;
1127
1128                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1129
1130                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1131                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1132                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1133                         if (s2 == d)
1134                                 M_IXOR(s1, d);
1135                         else {
1136                                 M_INTMOVE(s1, d);
1137                                 M_IXOR(s2, d);
1138                         }
1139                         emit_store_dst(jd, iptr, d);
1140                         break;
1141
1142                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1143                                       /* sx.val.i = constant                             */
1144
1145                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1146                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1147                         M_INTMOVE(s1, d);
1148                         M_IXOR_IMM(iptr->sx.val.i, d);
1149                         emit_store_dst(jd, iptr, d);
1150                         break;
1151
1152                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1153
1154                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1155                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1156                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1157                         if (s2 == d)
1158                                 M_LXOR(s1, d);
1159                         else {
1160                                 M_INTMOVE(s1, d);
1161                                 M_LXOR(s2, d);
1162                         }
1163                         emit_store_dst(jd, iptr, d);
1164                         break;
1165
1166                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1167                                       /* sx.val.l = constant                             */
1168
1169                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1170                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1171                         M_INTMOVE(s1, d);
1172                         if (IS_IMM32(iptr->sx.val.l))
1173                                 M_LXOR_IMM(iptr->sx.val.l, d);
1174                         else {
1175                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1176                                 M_LXOR(REG_ITMP2, d);
1177                         }
1178                         emit_store_dst(jd, iptr, d);
1179                         break;
1180
1181
1182                 /* floating operations ************************************************/
1183
1184                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1185
1186                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1187                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1188                         disp = dseg_add_s4(cd, 0x80000000);
1189                         M_FLTMOVE(s1, d);
1190                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1191                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1192                         emit_store_dst(jd, iptr, d);
1193                         break;
1194
1195                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1196
1197                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1198                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1199                         disp = dseg_add_s8(cd, 0x8000000000000000);
1200                         M_FLTMOVE(s1, d);
1201                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1202                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1203                         emit_store_dst(jd, iptr, d);
1204                         break;
1205
1206                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1207
1208                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1209                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1210                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1211                         if (s2 == d)
1212                                 M_FADD(s1, d);
1213                         else {
1214                                 M_FLTMOVE(s1, d);
1215                                 M_FADD(s2, d);
1216                         }
1217                         emit_store_dst(jd, iptr, d);
1218                         break;
1219
1220                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1221
1222                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1223                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1224                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1225                         if (s2 == d)
1226                                 M_DADD(s1, d);
1227                         else {
1228                                 M_FLTMOVE(s1, d);
1229                                 M_DADD(s2, d);
1230                         }
1231                         emit_store_dst(jd, iptr, d);
1232                         break;
1233
1234                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1235
1236                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1237                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1238                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1239                         if (s2 == d) {
1240                                 M_FLTMOVE(s2, REG_FTMP2);
1241                                 s2 = REG_FTMP2;
1242                         }
1243                         M_FLTMOVE(s1, d);
1244                         M_FSUB(s2, d);
1245                         emit_store_dst(jd, iptr, d);
1246                         break;
1247
1248                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1249
1250                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1251                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1252                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1253                         if (s2 == d) {
1254                                 M_FLTMOVE(s2, REG_FTMP2);
1255                                 s2 = REG_FTMP2;
1256                         }
1257                         M_FLTMOVE(s1, d);
1258                         M_DSUB(s2, d);
1259                         emit_store_dst(jd, iptr, d);
1260                         break;
1261
1262                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1263
1264                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1265                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1266                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1267                         if (s2 == d)
1268                                 M_FMUL(s1, d);
1269                         else {
1270                                 M_FLTMOVE(s1, d);
1271                                 M_FMUL(s2, d);
1272                         }
1273                         emit_store_dst(jd, iptr, d);
1274                         break;
1275
1276                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1277
1278                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1279                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1280                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1281                         if (s2 == d)
1282                                 M_DMUL(s1, d);
1283                         else {
1284                                 M_FLTMOVE(s1, d);
1285                                 M_DMUL(s2, d);
1286                         }
1287                         emit_store_dst(jd, iptr, d);
1288                         break;
1289
1290                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1291
1292                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1293                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1294                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1295                         if (s2 == d) {
1296                                 M_FLTMOVE(s2, REG_FTMP2);
1297                                 s2 = REG_FTMP2;
1298                         }
1299                         M_FLTMOVE(s1, d);
1300                         M_FDIV(s2, d);
1301                         emit_store_dst(jd, iptr, d);
1302                         break;
1303
1304                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1305
1306                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1307                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1308                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1309                         if (s2 == d) {
1310                                 M_FLTMOVE(s2, REG_FTMP2);
1311                                 s2 = REG_FTMP2;
1312                         }
1313                         M_FLTMOVE(s1, d);
1314                         M_DDIV(s2, d);
1315                         emit_store_dst(jd, iptr, d);
1316                         break;
1317
1318                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1319
1320                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1321                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1322                         M_CVTIF(s1, d);
1323                         emit_store_dst(jd, iptr, d);
1324                         break;
1325
1326                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1327
1328                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1329                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1330                         M_CVTID(s1, d);
1331                         emit_store_dst(jd, iptr, d);
1332                         break;
1333
1334                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1335
1336                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1337                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1338                         M_CVTLF(s1, d);
1339                         emit_store_dst(jd, iptr, d);
1340                         break;
1341
1342                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1343
1344                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1345                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1346                         M_CVTLD(s1, d);
1347                         emit_store_dst(jd, iptr, d);
1348                         break;
1349
1350                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1351
1352                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1353                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1354                         M_CVTFI(s1, d);
1355                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1356                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1357                                 ((REG_RESULT == d) ? 0 : 3);
1358                         M_BNE(disp);
1359                         M_FLTMOVE(s1, REG_FTMP1);
1360                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1361                         M_CALL(REG_ITMP2);
1362                         M_INTMOVE(REG_RESULT, d);
1363                         emit_store_dst(jd, iptr, d);
1364                         break;
1365
1366                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1367
1368                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1369                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1370                         M_CVTDI(s1, d);
1371                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1372                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1373                                 ((REG_RESULT == d) ? 0 : 3);
1374                         M_BNE(disp);
1375                         M_FLTMOVE(s1, REG_FTMP1);
1376                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1377                         M_CALL(REG_ITMP2);
1378                         M_INTMOVE(REG_RESULT, d);
1379                         emit_store_dst(jd, iptr, d);
1380                         break;
1381
1382                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1383
1384                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1385                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1386                         M_CVTFL(s1, d);
1387                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1388                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1389                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1390                                 ((REG_RESULT == d) ? 0 : 3);
1391                         M_BNE(disp);
1392                         M_FLTMOVE(s1, REG_FTMP1);
1393                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1394                         M_CALL(REG_ITMP2);
1395                         M_INTMOVE(REG_RESULT, d);
1396                         emit_store_dst(jd, iptr, d);
1397                         break;
1398
1399                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1400
1401                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1402                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1403                         M_CVTDL(s1, d);
1404                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1405                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1406                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1407                                 ((REG_RESULT == d) ? 0 : 3);
1408                         M_BNE(disp);
1409                         M_FLTMOVE(s1, REG_FTMP1);
1410                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1411                         M_CALL(REG_ITMP2);
1412                         M_INTMOVE(REG_RESULT, d);
1413                         emit_store_dst(jd, iptr, d);
1414                         break;
1415
1416                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1417
1418                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1419                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1420                         M_CVTFD(s1, d);
1421                         emit_store_dst(jd, iptr, d);
1422                         break;
1423
1424                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1425
1426                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1427                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1428                         M_CVTDF(s1, d);
1429                         emit_store_dst(jd, iptr, d);
1430                         break;
1431
1432                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1433                                           /* == => 0, < => 1, > => -1 */
1434
1435                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1436                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1437                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1438                         M_CLR(d);
1439                         M_MOV_IMM(1, REG_ITMP1);
1440                         M_MOV_IMM(-1, REG_ITMP2);
1441                         emit_ucomiss_reg_reg(cd, s1, s2);
1442                         M_CMOVULT(REG_ITMP1, d);
1443                         M_CMOVUGT(REG_ITMP2, d);
1444                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1445                         emit_store_dst(jd, iptr, d);
1446                         break;
1447
1448                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1449                                           /* == => 0, < => 1, > => -1 */
1450
1451                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1452                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1453                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1454                         M_CLR(d);
1455                         M_MOV_IMM(1, REG_ITMP1);
1456                         M_MOV_IMM(-1, REG_ITMP2);
1457                         emit_ucomiss_reg_reg(cd, s1, s2);
1458                         M_CMOVULT(REG_ITMP1, d);
1459                         M_CMOVUGT(REG_ITMP2, d);
1460                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1461                         emit_store_dst(jd, iptr, d);
1462                         break;
1463
1464                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1465                                           /* == => 0, < => 1, > => -1 */
1466
1467                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1468                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1469                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1470                         M_CLR(d);
1471                         M_MOV_IMM(1, REG_ITMP1);
1472                         M_MOV_IMM(-1, REG_ITMP2);
1473                         emit_ucomisd_reg_reg(cd, s1, s2);
1474                         M_CMOVULT(REG_ITMP1, d);
1475                         M_CMOVUGT(REG_ITMP2, d);
1476                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1477                         emit_store_dst(jd, iptr, d);
1478                         break;
1479
1480                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1481                                           /* == => 0, < => 1, > => -1 */
1482
1483                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1484                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1485                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1486                         M_CLR(d);
1487                         M_MOV_IMM(1, REG_ITMP1);
1488                         M_MOV_IMM(-1, REG_ITMP2);
1489                         emit_ucomisd_reg_reg(cd, s1, s2);
1490                         M_CMOVULT(REG_ITMP1, d);
1491                         M_CMOVUGT(REG_ITMP2, d);
1492                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1493                         emit_store_dst(jd, iptr, d);
1494                         break;
1495
1496
1497                 /* memory operations **************************************************/
1498
1499                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1500
1501                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1502                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1503                         /* implicit null-pointer check */
1504                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1505                         emit_store_dst(jd, iptr, d);
1506                         break;
1507
1508                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1509
1510                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1511                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1512                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1513                         /* implicit null-pointer check */
1514                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1515                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1516                         emit_store_dst(jd, iptr, d);
1517                         break;
1518
1519                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1520
1521                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1522                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1523                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1524                         /* implicit null-pointer check */
1525                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1526                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1527                         emit_store_dst(jd, iptr, d);
1528                         break;
1529
1530                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1531
1532                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1533                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1534                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1535                         /* implicit null-pointer check */
1536                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1537                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1538                         emit_store_dst(jd, iptr, d);
1539                         break;
1540
1541                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1542
1543                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1544                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1545                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1546                         /* implicit null-pointer check */
1547                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1548                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1549                         emit_store_dst(jd, iptr, d);
1550                         break;
1551
1552                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1553
1554                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1555                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1556                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1557                         /* implicit null-pointer check */
1558                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1559                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1560                         emit_store_dst(jd, iptr, d);
1561                         break;
1562
1563                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1564
1565                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1566                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1567                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1568                         /* implicit null-pointer check */
1569                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1570                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1571                         emit_store_dst(jd, iptr, d);
1572                         break;
1573
1574                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1575
1576                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1577                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1578                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1579                         /* implicit null-pointer check */
1580                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1581                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1582                         emit_store_dst(jd, iptr, d);
1583                         break;
1584
1585                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1586
1587                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1588                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1589                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1590                         /* implicit null-pointer check */
1591                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1592                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1593                         emit_store_dst(jd, iptr, d);
1594                         break;
1595
1596
1597                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1598
1599                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1600                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1601                         /* implicit null-pointer check */
1602                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1603                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1604                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1605                         break;
1606
1607                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1608
1609                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1610                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1611                         /* implicit null-pointer check */
1612                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1613                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1614                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1615                         break;
1616
1617                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1618
1619                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1620                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1621                         /* implicit null-pointer check */
1622                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1623                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1624                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1625                         break;
1626
1627                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1628
1629                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1630                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1631                         /* implicit null-pointer check */
1632                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1633                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1634                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
1635                         break;
1636
1637                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1638
1639                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1640                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1641                         /* implicit null-pointer check */
1642                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1643                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1644                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
1645                         break;
1646
1647                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1648
1649                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1650                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1651                         /* implicit null-pointer check */
1652                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1653                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1654                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
1655                         break;
1656
1657                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1658
1659                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1660                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1661                         /* implicit null-pointer check */
1662                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1663                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1664                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
1665                         break;
1666
1667                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1668
1669                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1670                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1671                         /* implicit null-pointer check */
1672                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1673                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1674
1675                         M_MOV(s1, REG_A0);
1676                         M_MOV(s3, REG_A1);
1677                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
1678                         M_CALL(REG_ITMP1);
1679                         emit_exception_check(cd, iptr);
1680
1681                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1682                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1683                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1684                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1685                         break;
1686
1687
1688                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1689
1690                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1691                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1692                         /* implicit null-pointer check */
1693                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1694                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1695                         break;
1696
1697                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1698
1699                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1700                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1701                         /* implicit null-pointer check */
1702                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1703                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray, data[0]), s1, s2, 1);
1704                         break;
1705
1706                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1707
1708                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1709                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1710                         /* implicit null-pointer check */
1711                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1712                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1713                         break;
1714
1715                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1716
1717                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1718                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1719                         /* implicit null-pointer check */
1720                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1721                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray, data[0]), s1, s2, 2);
1722                         break;
1723
1724                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1725
1726                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1727                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1728                         /* implicit null-pointer check */
1729                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1730
1731                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1732                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1733                         }
1734                         else {
1735                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1736                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
1737                         }
1738                         break;
1739
1740                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1741
1742                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1743                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1744                         /* implicit null-pointer check */
1745                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1746                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1747                         break;
1748
1749
1750                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1751
1752                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1753                                 uf        = iptr->sx.s23.s3.uf;
1754                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1755                                 disp      = dseg_add_unique_address(cd, NULL);
1756                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1757
1758                                 /* must be calculated before codegen_add_patch_ref */
1759
1760                                 if (opt_shownops)
1761                                         disp -= PATCHER_CALL_SIZE;
1762
1763 /*                              PROFILE_CYCLE_STOP; */
1764
1765                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1766
1767 /*                              PROFILE_CYCLE_START; */
1768                         }
1769                         else {
1770                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1771                                 fieldtype = fi->type;
1772                                 disp      = dseg_add_address(cd, &(fi->value));
1773                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1774
1775                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1776                                         PROFILE_CYCLE_STOP;
1777
1778                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1779
1780                                         PROFILE_CYCLE_START;
1781
1782                                         if (opt_shownops)
1783                                                 disp -= PATCHER_CALL_SIZE;
1784                                 }
1785                         }
1786
1787                         /* This approach is much faster than moving the field
1788                            address inline into a register. */
1789
1790                         M_ALD(REG_ITMP1, RIP, disp);
1791
1792                         switch (fieldtype) {
1793                         case TYPE_INT:
1794                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1795                                 M_ILD(d, REG_ITMP1, 0);
1796                                 break;
1797                         case TYPE_LNG:
1798                         case TYPE_ADR:
1799                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1800                                 M_LLD(d, REG_ITMP1, 0);
1801                                 break;
1802                         case TYPE_FLT:
1803                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1804                                 M_FLD(d, REG_ITMP1, 0);
1805                                 break;
1806                         case TYPE_DBL:
1807                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1808                                 M_DLD(d, REG_ITMP1, 0);
1809                                 break;
1810                         }
1811                         emit_store_dst(jd, iptr, d);
1812                         break;
1813
1814                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1815
1816                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1817                                 uf        = iptr->sx.s23.s3.uf;
1818                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1819                                 disp      = dseg_add_unique_address(cd, NULL);
1820                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1821
1822                                 /* must be calculated before codegen_add_patch_ref */
1823
1824                                 if (opt_shownops)
1825                                         disp -= PATCHER_CALL_SIZE;
1826
1827 /*                              PROFILE_CYCLE_STOP; */
1828
1829                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1830
1831 /*                              PROFILE_CYCLE_START; */
1832                         }
1833                         else {
1834                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1835                                 fieldtype = fi->type;
1836                                 disp      = dseg_add_address(cd, &(fi->value));
1837                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1838
1839                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1840                                         PROFILE_CYCLE_STOP;
1841
1842                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1843
1844                                         PROFILE_CYCLE_START;
1845
1846                                         if (opt_shownops)
1847                                                 disp -= PATCHER_CALL_SIZE;
1848                                 }
1849                         }
1850
1851                         /* This approach is much faster than moving the field
1852                            address inline into a register. */
1853
1854                         M_ALD(REG_ITMP1, RIP, disp);
1855
1856                         switch (fieldtype) {
1857                         case TYPE_INT:
1858                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1859                                 M_IST(s1, REG_ITMP1, 0);
1860                                 break;
1861                         case TYPE_LNG:
1862                         case TYPE_ADR:
1863                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1864                                 M_LST(s1, REG_ITMP1, 0);
1865                                 break;
1866                         case TYPE_FLT:
1867                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1868                                 M_FST(s1, REG_ITMP1, 0);
1869                                 break;
1870                         case TYPE_DBL:
1871                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1872                                 M_DST(s1, REG_ITMP1, 0);
1873                                 break;
1874                         }
1875                         break;
1876
1877                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1878                                           /* val = value (in current instruction)     */
1879                                           /* following NOP)                           */
1880
1881                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1882                                 uf        = iptr->sx.s23.s3.uf;
1883                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1884                                 disp      = dseg_add_unique_address(cd, NULL);
1885                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1886
1887                                 /* must be calculated before codegen_add_patch_ref */
1888
1889                                 if (opt_shownops)
1890                                         disp -= PATCHER_CALL_SIZE;
1891
1892 /*                              PROFILE_CYCLE_STOP; */
1893
1894                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1895
1896 /*                              PROFILE_CYCLE_START; */
1897                         }
1898                         else {
1899                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1900                                 fieldtype = fi->type;
1901                                 disp      = dseg_add_address(cd, &(fi->value));
1902                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1903
1904                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1905                                         PROFILE_CYCLE_STOP;
1906
1907                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1908
1909                                         PROFILE_CYCLE_START;
1910
1911                                         if (opt_shownops)
1912                                                 disp -= PATCHER_CALL_SIZE;
1913                                 }
1914                         }
1915
1916                         /* This approach is much faster than moving the field
1917                            address inline into a register. */
1918
1919                         M_ALD(REG_ITMP1, RIP, disp);
1920
1921                         switch (fieldtype) {
1922                         case TYPE_INT:
1923                         case TYPE_FLT:
1924                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1925                                 break;
1926                         case TYPE_LNG:
1927                         case TYPE_ADR:
1928                         case TYPE_DBL:
1929                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1930                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1931                                 else {
1932                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1933                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1934                                 }
1935                                 break;
1936                         }
1937                         break;
1938
1939                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1940
1941                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1942
1943                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1944                                 uf        = iptr->sx.s23.s3.uf;
1945                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1946                                 disp      = 0;
1947
1948 /*                              PROFILE_CYCLE_STOP; */
1949
1950                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1951
1952 /*                              PROFILE_CYCLE_START; */
1953                         }
1954                         else {
1955                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1956                                 fieldtype = fi->type;
1957                                 disp      = fi->offset;
1958                         }
1959
1960                         /* implicit null-pointer check */
1961                         switch (fieldtype) {
1962                         case TYPE_INT:
1963                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1964                                 M_ILD32(d, s1, disp);
1965                                 break;
1966                         case TYPE_LNG:
1967                         case TYPE_ADR:
1968                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1969                                 M_LLD32(d, s1, disp);
1970                                 break;
1971                         case TYPE_FLT:
1972                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1973                                 M_FLD32(d, s1, disp);
1974                                 break;
1975                         case TYPE_DBL:
1976                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1977                                 M_DLD32(d, s1, disp);
1978                                 break;
1979                         }
1980                         emit_store_dst(jd, iptr, d);
1981                         break;
1982
1983                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1984
1985                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1986                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1987
1988                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1989                                 uf        = iptr->sx.s23.s3.uf;
1990                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1991                                 disp      = 0;
1992
1993 /*                              PROFILE_CYCLE_STOP; */
1994
1995                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1996
1997 /*                              PROFILE_CYCLE_START; */
1998                         }
1999                         else {
2000                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2001                                 fieldtype = fi->type;
2002                                 disp      = fi->offset;
2003                         }
2004
2005                         /* implicit null-pointer check */
2006                         switch (fieldtype) {
2007                         case TYPE_INT:
2008                                 M_IST32(s2, s1, disp);
2009                                 break;
2010                         case TYPE_LNG:
2011                         case TYPE_ADR:
2012                                 M_LST32(s2, s1, disp);
2013                                 break;
2014                         case TYPE_FLT:
2015                                 M_FST32(s2, s1, disp);
2016                                 break;
2017                         case TYPE_DBL:
2018                                 M_DST32(s2, s1, disp);
2019                                 break;
2020                         }
2021                         break;
2022
2023                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2024                                           /* val = value (in current instruction)     */
2025                                           /* following NOP)                           */
2026
2027                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2028
2029                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2030                                 uf        = iptr->sx.s23.s3.uf;
2031                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2032                                 disp      = 0;
2033
2034 /*                              PROFILE_CYCLE_STOP; */
2035
2036                                 codegen_add_patch_ref(cd, PATCHER_putfieldconst, uf, 0);
2037
2038 /*                              PROFILE_CYCLE_START; */
2039                         }
2040                         else {
2041                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2042                                 fieldtype = fi->type;
2043                                 disp      = fi->offset;
2044                         }
2045
2046                         /* implicit null-pointer check */
2047                         switch (fieldtype) {
2048                         case TYPE_INT:
2049                         case TYPE_FLT:
2050                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2051                                 break;
2052                         case TYPE_LNG:
2053                         case TYPE_ADR:
2054                         case TYPE_DBL:
2055                                 /* XXX why no check for IS_IMM32? */
2056                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2057                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2058                                 break;
2059                         }
2060                         break;
2061
2062
2063                 /* branch operations **************************************************/
2064
2065                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2066
2067                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2068                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2069
2070                         PROFILE_CYCLE_STOP;
2071
2072 #ifdef ENABLE_VERIFIER
2073                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2074                                 uc = iptr->sx.s23.s2.uc;
2075
2076                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2077                         }
2078 #endif /* ENABLE_VERIFIER */
2079
2080                         M_CALL_IMM(0);                            /* passing exception pc */
2081                         M_POP(REG_ITMP2_XPC);
2082
2083                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2084                         M_JMP(REG_ITMP3);
2085                         break;
2086
2087                 case ICMD_GOTO:         /* ... ==> ...                                */
2088                 case ICMD_RET:
2089
2090                         emit_br(cd, iptr->dst.block);
2091                         ALIGNCODENOP;
2092                         break;
2093
2094                 case ICMD_JSR:          /* ... ==> ...                                */
2095
2096                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2097                         ALIGNCODENOP;
2098                         break;
2099
2100                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2101                 case ICMD_IFNONNULL:
2102
2103                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2104                         M_TEST(s1);
2105                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2106                         break;
2107
2108                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2109                 case ICMD_IFLT:
2110                 case ICMD_IFLE:
2111                 case ICMD_IFNE:
2112                 case ICMD_IFGT:
2113                 case ICMD_IFGE:
2114
2115                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2116                         M_ICMP_IMM(iptr->sx.val.i, s1);
2117                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2118                         break;
2119
2120                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2121                 case ICMD_IF_LNE:
2122                 case ICMD_IF_LLT:
2123                 case ICMD_IF_LGE:
2124                 case ICMD_IF_LGT:
2125                 case ICMD_IF_LLE:
2126
2127                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2128                         if (IS_IMM32(iptr->sx.val.l))
2129                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2130                         else {
2131                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2132                                 M_LCMP(REG_ITMP2, s1);
2133                         }
2134                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2135                         break;
2136
2137                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2138                 case ICMD_IF_ICMPNE:
2139                 case ICMD_IF_ICMPLT:
2140                 case ICMD_IF_ICMPGE:
2141                 case ICMD_IF_ICMPGT:
2142                 case ICMD_IF_ICMPLE:
2143
2144                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2145                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2146                         M_ICMP(s2, s1);
2147                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2148                         break;
2149
2150                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2151                 case ICMD_IF_ACMPNE:
2152
2153                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2154                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2155                         M_LCMP(s2, s1);
2156                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2157                         break;
2158
2159                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2160                 case ICMD_IF_LCMPNE:
2161                 case ICMD_IF_LCMPLT:
2162                 case ICMD_IF_LCMPGE:
2163                 case ICMD_IF_LCMPGT:
2164                 case ICMD_IF_LCMPLE:
2165
2166                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2167                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2168                         M_LCMP(s2, s1);
2169                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2170                         break;
2171
2172                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2173                 case ICMD_LRETURN:
2174
2175                         REPLACEMENT_POINT_RETURN(cd, iptr);
2176                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2177                         M_INTMOVE(s1, REG_RESULT);
2178                         goto nowperformreturn;
2179
2180                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2181
2182                         REPLACEMENT_POINT_RETURN(cd, iptr);
2183                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2184                         M_INTMOVE(s1, REG_RESULT);
2185
2186 #ifdef ENABLE_VERIFIER
2187                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2188                                 uc = iptr->sx.s23.s2.uc;
2189
2190                                 PROFILE_CYCLE_STOP;
2191
2192                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2193
2194                                 PROFILE_CYCLE_START;
2195                         }
2196 #endif /* ENABLE_VERIFIER */
2197                         goto nowperformreturn;
2198
2199                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2200                 case ICMD_DRETURN:
2201
2202                         REPLACEMENT_POINT_RETURN(cd, iptr);
2203                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2204                         M_FLTMOVE(s1, REG_FRESULT);
2205                         goto nowperformreturn;
2206
2207                 case ICMD_RETURN:      /* ...  ==> ...                                */
2208
2209                         REPLACEMENT_POINT_RETURN(cd, iptr);
2210
2211 nowperformreturn:
2212                         {
2213                         s4 i, p;
2214
2215                         p = cd->stackframesize;
2216
2217 #if !defined(NDEBUG)
2218                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2219                                 emit_verbosecall_exit(jd);
2220 #endif /* !defined(NDEBUG) */
2221
2222 #if defined(ENABLE_THREADS)
2223                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2224                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2225
2226                                 /* we need to save the proper return value */
2227                                 switch (iptr->opc) {
2228                                 case ICMD_IRETURN:
2229                                 case ICMD_ARETURN:
2230                                 case ICMD_LRETURN:
2231                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2232                                         break;
2233                                 case ICMD_FRETURN:
2234                                 case ICMD_DRETURN:
2235                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2236                                         break;
2237                                 }
2238
2239                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2240                                 M_CALL(REG_ITMP1);
2241
2242                                 /* and now restore the proper return value */
2243                                 switch (iptr->opc) {
2244                                 case ICMD_IRETURN:
2245                                 case ICMD_ARETURN:
2246                                 case ICMD_LRETURN:
2247                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2248                                         break;
2249                                 case ICMD_FRETURN:
2250                                 case ICMD_DRETURN:
2251                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2252                                         break;
2253                                 }
2254                         }
2255 #endif
2256
2257                         /* restore saved registers */
2258
2259                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2260                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2261                         }
2262                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2263                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2264                         }
2265
2266                         /* deallocate stack */
2267
2268                         if (cd->stackframesize)
2269                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2270
2271                         /* generate method profiling code */
2272
2273                         PROFILE_CYCLE_STOP;
2274
2275                         M_RET;
2276                         }
2277                         break;
2278
2279
2280                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2281                         {
2282                                 s4 i, l;
2283                                 branch_target_t *table;
2284
2285                                 table = iptr->dst.table;
2286
2287                                 l = iptr->sx.s23.s2.tablelow;
2288                                 i = iptr->sx.s23.s3.tablehigh;
2289
2290                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2291                                 M_INTMOVE(s1, REG_ITMP1);
2292
2293                                 if (l != 0)
2294                                         M_ISUB_IMM(l, REG_ITMP1);
2295
2296                                 /* number of targets */
2297                                 i = i - l + 1;
2298
2299                 /* range check */
2300
2301                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2302                                 emit_bugt(cd, table[0].block);
2303
2304                                 /* build jump table top down and use address of lowest entry */
2305
2306                                 table += i;
2307
2308                                 while (--i >= 0) {
2309                                         dseg_add_target(cd, table->block);
2310                                         --table;
2311                                 }
2312
2313                                 /* length of dataseg after last dseg_add_target is used
2314                                    by load */
2315
2316                                 M_MOV_IMM(0, REG_ITMP2);
2317                                 dseg_adddata(cd);
2318                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2319                                 M_JMP(REG_ITMP1);
2320                         }
2321                         break;
2322
2323
2324                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2325                         {
2326                                 s4 i;
2327                                 lookup_target_t *lookup;
2328
2329                                 lookup = iptr->dst.lookup;
2330
2331                                 i = iptr->sx.s23.s2.lookupcount;
2332
2333                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2334                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2335
2336                                 while (--i >= 0) {
2337                                         M_ICMP_IMM(lookup->value, s1);
2338                                         emit_beq(cd, lookup->target.block);
2339                                         lookup++;
2340                                 }
2341
2342                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2343                                 ALIGNCODENOP;
2344                         }
2345                         break;
2346
2347
2348                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2349
2350                         bte = iptr->sx.s23.s3.bte;
2351                         md  = bte->md;
2352                         goto gen_method;
2353
2354                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2355
2356                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2357                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2358                 case ICMD_INVOKEINTERFACE:
2359
2360                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2361
2362                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2363                                 lm = NULL;
2364                                 um = iptr->sx.s23.s3.um;
2365                                 md = um->methodref->parseddesc.md;
2366                         }
2367                         else {
2368                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2369                                 um = NULL;
2370                                 md = lm->parseddesc;
2371                         }
2372
2373 gen_method:
2374                         s3 = md->paramcount;
2375
2376                         MCODECHECK((20 * s3) + 128);
2377
2378                         /* copy arguments to registers or stack location */
2379
2380                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2381                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2382                                 d   = md->params[s3].regoff;
2383
2384                                 /* already preallocated (ARGVAR)? */
2385
2386                                 if (var->flags & PREALLOC)
2387                                         continue;
2388
2389                                 if (IS_INT_LNG_TYPE(var->type)) {
2390                                         if (!md->params[s3].inmemory) {
2391                                                 s1 = emit_load(jd, iptr, var, d);
2392                                                 M_INTMOVE(s1, d);
2393                                         }
2394                                         else {
2395                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2396                                                 M_LST(s1, REG_SP, d * 8);
2397                                         }
2398                                 }
2399                                 else {
2400                                         if (!md->params[s3].inmemory) {
2401                                                 s1 = emit_load(jd, iptr, var, d);
2402                                                 M_FLTMOVE(s1, d);
2403                                         }
2404                                         else {
2405                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2406
2407                                                 if (IS_2_WORD_TYPE(var->type))
2408                                                         M_DST(s1, REG_SP, d * 8);
2409                                                 else
2410                                                         M_FST(s1, REG_SP, d * 8);
2411                                         }
2412                                 }
2413                         }
2414
2415                         /* generate method profiling code */
2416
2417                         PROFILE_CYCLE_STOP;
2418
2419                         switch (iptr->opc) {
2420                         case ICMD_BUILTIN:
2421                                 if (bte->stub == NULL) {
2422                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2423                                 } else {
2424                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2425                                 }
2426                                 M_CALL(REG_ITMP1);
2427
2428                                 emit_exception_check(cd, iptr);
2429                                 break;
2430
2431                         case ICMD_INVOKESPECIAL:
2432                                 emit_nullpointer_check(cd, iptr, REG_A0);
2433                                 /* fall through */
2434
2435                         case ICMD_INVOKESTATIC:
2436                                 if (lm == NULL) {
2437                                         disp = dseg_add_unique_address(cd, NULL);
2438                                         disp = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
2439
2440                                         /* must be calculated before codegen_add_patch_ref */
2441
2442                                         if (opt_shownops)
2443                                                 disp -= PATCHER_CALL_SIZE;
2444
2445                                         codegen_add_patch_ref(cd, PATCHER_invokestatic_special,
2446                                                                                   um, disp);
2447
2448 /*                                      a = 0; */
2449                                 }
2450                                 else {
2451                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2452                                         disp = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
2453
2454 /*                                      a = (ptrint) lm->stubroutine; */
2455                                 }
2456
2457 /*                              M_MOV_IMM(a, REG_ITMP2); */
2458                                 M_ALD(REG_ITMP2, RIP, disp);
2459                                 M_CALL(REG_ITMP2);
2460                                 break;
2461
2462                         case ICMD_INVOKEVIRTUAL:
2463                                 if (lm == NULL) {
2464                                         codegen_add_patch_ref(cd, PATCHER_invokevirtual, um, 0);
2465
2466                                         s1 = 0;
2467                                 }
2468                                 else {
2469                                         s1 = OFFSET(vftbl_t, table[0]) +
2470                                                 sizeof(methodptr) * lm->vftblindex;
2471                                 }
2472
2473                                 /* implicit null-pointer check */
2474                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_objectheader, vftbl));
2475                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2476                                 M_CALL(REG_ITMP3);
2477                                 break;
2478
2479                         case ICMD_INVOKEINTERFACE:
2480                                 if (lm == NULL) {
2481                                         codegen_add_patch_ref(cd, PATCHER_invokeinterface, um, 0);
2482
2483                                         s1 = 0;
2484                                         s2 = 0;
2485                                 }
2486                                 else {
2487                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2488                                                 sizeof(methodptr) * lm->class->index;
2489
2490                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2491                                 }
2492
2493                                 /* implicit null-pointer check */
2494                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_objectheader, vftbl));
2495                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2496                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2497                                 M_CALL(REG_ITMP3);
2498                                 break;
2499                         }
2500
2501                         /* generate method profiling code */
2502
2503                         PROFILE_CYCLE_START;
2504
2505                         /* store size of call code in replacement point */
2506
2507                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2508
2509                         /* store return value */
2510
2511                         switch (md->returntype.type) {
2512                         case TYPE_INT:
2513                         case TYPE_LNG:
2514                         case TYPE_ADR:
2515                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2516                                 M_INTMOVE(REG_RESULT, s1);
2517                                 emit_store_dst(jd, iptr, s1);
2518                                 break;
2519                         case TYPE_FLT:
2520                         case TYPE_DBL:
2521                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2522                                 M_FLTMOVE(REG_FRESULT, s1);
2523                                 emit_store_dst(jd, iptr, s1);
2524                                 break;
2525                         default:
2526                                 /* TYPE_VOID */
2527                                 break;
2528                         }
2529                         break;
2530
2531
2532                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2533
2534                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2535                                 /* object type cast-check */
2536
2537                                 classinfo *super;
2538                                 vftbl_t   *supervftbl;
2539                                 s4         superindex;
2540
2541                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2542                                         super      = NULL;
2543                                         superindex = 0;
2544                                         supervftbl = NULL;
2545                                 }
2546                                 else {
2547                                         super      = iptr->sx.s23.s3.c.cls;
2548                                         superindex = super->index;
2549                                         supervftbl = super->vftbl;
2550                                 }
2551
2552 #if defined(ENABLE_THREADS)
2553                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
2554 #endif
2555                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2556
2557                                 /* if class is not resolved, check which code to call */
2558
2559                                 if (super == NULL) {
2560                                         M_TEST(s1);
2561                                         emit_label_beq(cd, BRANCH_LABEL_1);
2562
2563                                         codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2564                                                                                   iptr->sx.s23.s3.c.ref, 0);
2565
2566                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2567                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2568                                         emit_label_beq(cd, BRANCH_LABEL_2);
2569                                 }
2570
2571                                 /* interface checkcast code */
2572
2573                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2574                                         if (super != NULL) {
2575                                                 M_TEST(s1);
2576                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2577                                         }
2578
2579                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2580
2581                                         if (super == NULL) {
2582                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_interface,
2583                                                                                           iptr->sx.s23.s3.c.ref,
2584                                                                                           0);
2585                                         }
2586
2587                                         M_ILD32(REG_ITMP3,
2588                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2589                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2590                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2591
2592                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2593                                                         OFFSET(vftbl_t, interfacetable[0]) -
2594                                                         superindex * sizeof(methodptr*));
2595                                         M_TEST(REG_ITMP3);
2596                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2597
2598                                         if (super == NULL)
2599                                                 emit_label_br(cd, BRANCH_LABEL_4);
2600                                         else
2601                                                 emit_label(cd, BRANCH_LABEL_3);
2602                                 }
2603
2604                                 /* class checkcast code */
2605
2606                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2607                                         if (super == NULL) {
2608                                                 emit_label(cd, BRANCH_LABEL_2);
2609                                         }
2610                                         else {
2611                                                 M_TEST(s1);
2612                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2613                                         }
2614
2615                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2616
2617                                         if (super == NULL) {
2618                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_class,
2619                                                                                           iptr->sx.s23.s3.c.ref,
2620                                                                                           0);
2621                                         }
2622
2623                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2624 #if defined(ENABLE_THREADS)
2625                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
2626 #endif
2627                                         M_ILD32(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2628
2629                                         /*                                      if (s1 != REG_ITMP1) { */
2630                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2631                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2632                                         /*                                                                                              REG_ITMP1); */
2633                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2634                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2635                                         /*                                                                                              REG_ITMP3); */
2636                                         /*  #if defined(ENABLE_THREADS) */
2637                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2638                                         /*  #endif */
2639                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2640
2641                                         /*                                      } else { */
2642
2643                                         M_ILD32(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
2644                                         M_ISUB(REG_ITMP3, REG_ITMP2);
2645                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2646                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2647                                         /*                                      } */
2648 #if defined(ENABLE_THREADS)
2649                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
2650 #endif
2651                                         M_ICMP(REG_ITMP3, REG_ITMP2);
2652                                         emit_classcast_check(cd, iptr, BRANCH_UGT, REG_ITMP3, s1);
2653
2654                                         if (super != NULL)
2655                                                 emit_label(cd, BRANCH_LABEL_5);
2656                                 }
2657
2658                                 if (super == NULL) {
2659                                         emit_label(cd, BRANCH_LABEL_1);
2660                                         emit_label(cd, BRANCH_LABEL_4);
2661                                 }
2662
2663                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2664                         }
2665                         else {
2666                                 /* array type cast-check */
2667
2668                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2669                                 M_INTMOVE(s1, REG_A0);
2670
2671                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2672                                         codegen_add_patch_ref(cd, PATCHER_builtin_arraycheckcast,
2673                                                                                   iptr->sx.s23.s3.c.ref, 0);
2674                                 }
2675
2676                                 M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2677                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2678                                 M_CALL(REG_ITMP1);
2679
2680                                 /* s1 may have been destroyed over the function call */
2681                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2682                                 M_TEST(REG_RESULT);
2683                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2684
2685                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2686                         }
2687
2688                         M_INTMOVE(s1, d);
2689                         emit_store_dst(jd, iptr, d);
2690                         break;
2691
2692                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2693
2694                         {
2695                         classinfo *super;
2696                         vftbl_t   *supervftbl;
2697                         s4         superindex;
2698
2699                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2700                                 super      = NULL;
2701                                 superindex = 0;
2702                                 supervftbl = NULL;
2703
2704                         } else {
2705                                 super      = iptr->sx.s23.s3.c.cls;
2706                                 superindex = super->index;
2707                                 supervftbl = super->vftbl;
2708                         }
2709
2710 #if defined(ENABLE_THREADS)
2711             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
2712 #endif
2713
2714                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2715                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2716
2717                         if (s1 == d) {
2718                                 M_INTMOVE(s1, REG_ITMP1);
2719                                 s1 = REG_ITMP1;
2720                         }
2721
2722                         M_CLR(d);
2723
2724                         /* if class is not resolved, check which code to call */
2725
2726                         if (super == NULL) {
2727                                 M_TEST(s1);
2728                                 emit_label_beq(cd, BRANCH_LABEL_1);
2729
2730                                 codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2731                                                                           iptr->sx.s23.s3.c.ref, 0);
2732
2733                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2734                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2735                                 emit_label_beq(cd, BRANCH_LABEL_2);
2736                         }
2737
2738                         /* interface instanceof code */
2739
2740                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2741                                 if (super != NULL) {
2742                                         M_TEST(s1);
2743                                         emit_label_beq(cd, BRANCH_LABEL_3);
2744                                 }
2745
2746                                 M_ALD(REG_ITMP1, s1, OFFSET(java_objectheader, vftbl));
2747
2748                                 if (super == NULL) {
2749                                         codegen_add_patch_ref(cd, PATCHER_instanceof_interface,
2750                                                                                   iptr->sx.s23.s3.c.ref, 0);
2751                                 }
2752
2753                                 M_ILD32(REG_ITMP3,
2754                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2755                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2756
2757                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2758
2759                                 M_BLE(a);
2760                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2761                                                 OFFSET(vftbl_t, interfacetable[0]) -
2762                                                 superindex * sizeof(methodptr*));
2763                                 M_TEST(REG_ITMP1);
2764                                 M_SETNE(d);
2765
2766                                 if (super == NULL)
2767                                         emit_label_br(cd, BRANCH_LABEL_4);
2768                                 else
2769                                         emit_label(cd, BRANCH_LABEL_3);
2770                         }
2771
2772                         /* class instanceof code */
2773
2774                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2775                                 if (super == NULL) {
2776                                         emit_label(cd, BRANCH_LABEL_2);
2777                                 }
2778                                 else {
2779                                         M_TEST(s1);
2780                                         emit_label_beq(cd, BRANCH_LABEL_5);
2781                                 }
2782
2783                                 M_ALD(REG_ITMP1, s1, OFFSET(java_objectheader, vftbl));
2784
2785                                 if (super == NULL) {
2786                                         codegen_add_patch_ref(cd, PATCHER_instanceof_class,
2787                                                                                   iptr->sx.s23.s3.c.ref, 0);
2788                                 }
2789
2790                                 M_MOV_IMM(supervftbl, REG_ITMP2);
2791
2792 #if defined(ENABLE_THREADS)
2793                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
2794 #endif
2795
2796                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
2797                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2798                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2799
2800 #if defined(ENABLE_THREADS)
2801                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
2802 #endif
2803
2804                                 M_ISUB(REG_ITMP2, REG_ITMP1);
2805                                 M_CLR(d); /* may be REG_ITMP2 */
2806                                 M_ICMP(REG_ITMP3, REG_ITMP1);
2807                                 M_SETULE(d);
2808
2809                                 if (super != NULL)
2810                                         emit_label(cd, BRANCH_LABEL_5);
2811                         }
2812
2813                         if (super == NULL) {
2814                                 emit_label(cd, BRANCH_LABEL_1);
2815                                 emit_label(cd, BRANCH_LABEL_4);
2816                         }
2817
2818                         emit_store_dst(jd, iptr, d);
2819                         }
2820                         break;
2821
2822                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2823
2824                         /* check for negative sizes and copy sizes to stack if necessary  */
2825
2826                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2827
2828                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2829
2830                                 /* copy SAVEDVAR sizes to stack */
2831                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2832
2833                                 /* Already Preallocated? */
2834                                 if (!(var->flags & PREALLOC)) {
2835                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2836                                         M_LST(s2, REG_SP, s1 * 8);
2837                                 }
2838                         }
2839
2840                         /* is a patcher function set? */
2841
2842                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2843                                 codegen_add_patch_ref(cd, PATCHER_builtin_multianewarray,
2844                                                                           iptr->sx.s23.s3.c.ref, 0);
2845                         }
2846
2847                         /* a0 = dimension count */
2848
2849                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2850
2851                         /* a1 = classinfo */
2852
2853                         M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2854
2855                         /* a2 = pointer to dimensions = stack pointer */
2856
2857                         M_MOV(REG_SP, REG_A2);
2858
2859                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2860                         M_CALL(REG_ITMP1);
2861
2862                         /* check for exception before result assignment */
2863
2864                         emit_exception_check(cd, iptr);
2865
2866                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2867                         M_INTMOVE(REG_RESULT, s1);
2868                         emit_store_dst(jd, iptr, s1);
2869                         break;
2870
2871                 default:
2872                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2873                                                                                    iptr->opc);
2874                         return false;
2875         } /* switch */
2876
2877         } /* for instruction */
2878
2879         MCODECHECK(512); /* XXX require a lower number? */
2880
2881         /* At the end of a basic block we may have to append some nops,
2882            because the patcher stub calling code might be longer than the
2883            actual instruction. So codepatching does not change the
2884            following block unintentionally. */
2885
2886         if (cd->mcodeptr < cd->lastmcodeptr) {
2887                 while (cd->mcodeptr < cd->lastmcodeptr) {
2888                         M_NOP;
2889                 }
2890         }
2891
2892         } /* if (bptr -> flags >= BBREACHED) */
2893         } /* for basic block */
2894
2895         dseg_createlinenumbertable(cd);
2896
2897         /* generate stubs */
2898
2899         emit_patcher_stubs(jd);
2900         REPLACEMENT_EMIT_STUBS(jd);
2901
2902         /* everything's ok */
2903
2904         return true;
2905 }
2906
2907
2908 /* codegen_emit_stub_compiler **************************************************
2909
2910    Emit a stub routine which calls the compiler.
2911
2912 *******************************************************************************/
2913
2914 void codegen_emit_stub_compiler(jitdata *jd)
2915 {
2916         methodinfo  *m;
2917         codegendata *cd;
2918
2919         /* get required compiler data */
2920
2921         m  = jd->m;
2922         cd = jd->cd;
2923
2924         /* code for the stub */
2925
2926         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
2927         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
2928         M_JMP(REG_ITMP3);
2929 }
2930
2931
2932 /* codegen_emit_stub_builtin ***************************************************
2933
2934    Creates a stub routine which calls a builtin function.
2935
2936 *******************************************************************************/
2937
2938 void codegen_emit_stub_builtin(jitdata *jd, builtintable_entry *bte)
2939 {
2940         codeinfo    *code;
2941         codegendata *cd;
2942         methoddesc  *md;
2943         s4           i, j;
2944         s4           s1, disp;
2945
2946         /* get required compiler data */
2947
2948         code = jd->code;
2949         cd   = jd->cd;
2950
2951         md = bte->md;
2952
2953         /* calculate stack frame size */
2954
2955         cd->stackframesize =
2956                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
2957                 1;                                                    /* return value */
2958
2959         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
2960
2961         /* create method header */
2962
2963         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2964         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2965         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
2966         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2967         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2968         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2969         (void) dseg_addlinenumbertablesize(cd);
2970         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
2971
2972         /* generate stub code */
2973
2974         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2975
2976 #if defined(ENABLE_GC_CACAO)
2977         /* Save callee saved integer registers in stackframeinfo (GC may
2978            need to recover them during a collection). */
2979
2980         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
2981                 OFFSET(stackframeinfo, intregs);
2982
2983         for (i = 0; i < INT_SAV_CNT; i++)
2984                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2985 #endif
2986
2987         /* save integer and float argument registers */
2988
2989         for (i = 0, j = 0; i < md->paramcount; i++) {
2990                 if (!md->params[i].inmemory) {
2991                         s1 = md->params[i].regoff;
2992
2993                         switch (md->paramtypes[i].type) {
2994                         case TYPE_INT:
2995                         case TYPE_LNG:
2996                         case TYPE_ADR:
2997                                 M_LST(s1, REG_SP, j * 8);
2998                                 break;
2999                         case TYPE_FLT:
3000                         case TYPE_DBL:
3001                                 M_DST(s1, REG_SP, j * 8);
3002                                 break;
3003                         }
3004
3005                         j++;
3006                 }
3007         }
3008
3009         /* create dynamic stack info */
3010
3011         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3012         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3013         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3014         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3015         M_MOV_IMM(codegen_stub_builtin_enter, REG_ITMP1);
3016         M_CALL(REG_ITMP1);
3017
3018         /* restore integer and float argument registers */
3019
3020         for (i = 0, j = 0; i < md->paramcount; i++) {
3021                 if (!md->params[i].inmemory) {
3022                         s1 = md->params[i].regoff;
3023
3024                         switch (md->paramtypes[i].type) {
3025                         case TYPE_INT:
3026                         case TYPE_LNG:
3027                         case TYPE_ADR:
3028                                 M_LLD(s1, REG_SP, j * 8);
3029                                 break;
3030                         case TYPE_FLT:
3031                         case TYPE_DBL:
3032                                 M_DLD(s1, REG_SP, j * 8);
3033                                 break;
3034                         }
3035
3036                         j++;
3037                 }
3038         }
3039
3040         /* call the builtin function */
3041
3042         M_MOV_IMM(bte->fp, REG_ITMP3);
3043         M_CALL(REG_ITMP3);
3044
3045         /* save return value */
3046
3047         switch (md->returntype.type) {
3048         case TYPE_INT:
3049         case TYPE_LNG:
3050         case TYPE_ADR:
3051                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3052                 break;
3053         case TYPE_FLT:
3054         case TYPE_DBL:
3055                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3056                 break;
3057         case TYPE_VOID:
3058                 break;
3059         }
3060
3061         /* remove native stackframe info */
3062
3063         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3064         M_MOV_IMM(codegen_stub_builtin_exit, REG_ITMP1);
3065         M_CALL(REG_ITMP1);
3066
3067         /* restore return value */
3068
3069         switch (md->returntype.type) {
3070         case TYPE_INT:
3071         case TYPE_LNG:
3072         case TYPE_ADR:
3073                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3074                 break;
3075         case TYPE_FLT:
3076         case TYPE_DBL:
3077                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3078                 break;
3079         case TYPE_VOID:
3080                 break;
3081         }
3082
3083 #if defined(ENABLE_GC_CACAO)
3084         /* Restore callee saved integer registers from stackframeinfo (GC
3085            might have modified them during a collection). */
3086
3087         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3088                 OFFSET(stackframeinfo, intregs);
3089
3090         for (i = 0; i < INT_SAV_CNT; i++)
3091                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3092 #endif
3093
3094         /* remove stackframe */
3095
3096         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3097         M_RET;
3098 }
3099
3100
3101 /* codegen_emit_stub_native ****************************************************
3102
3103    Emits a stub routine which calls a native method.
3104
3105 *******************************************************************************/
3106
3107 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f)
3108 {
3109         methodinfo  *m;
3110         codeinfo    *code;
3111         codegendata *cd;
3112         methoddesc  *md;
3113         s4           nativeparams;
3114         s4           i, j;
3115         s4           t;
3116         s4           s1, s2, disp;
3117
3118         /* get required compiler data */
3119
3120         m    = jd->m;
3121         code = jd->code;
3122         cd   = jd->cd;
3123
3124         /* initialize variables */
3125
3126         md = m->parseddesc;
3127         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
3128
3129         /* calculate stack frame size */
3130
3131         cd->stackframesize =
3132                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
3133                 sizeof(localref_table) / SIZEOF_VOID_P +
3134                 md->paramcount +
3135                 1 +                       /* functionptr, TODO: store in data segment */
3136                 nmd->memuse;
3137
3138         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
3139
3140         /* create method header */
3141
3142         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
3143         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
3144         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
3145         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
3146         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
3147         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
3148         (void) dseg_addlinenumbertablesize(cd);
3149         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
3150
3151 #if defined(ENABLE_PROFILING)
3152         /* generate native method profiling code */
3153
3154         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3155                 /* count frequency */
3156
3157                 M_MOV_IMM(code, REG_ITMP3);
3158                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3159         }
3160 #endif
3161
3162         /* generate stub code */
3163
3164         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3165
3166 #if !defined(NDEBUG)
3167         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3168                 emit_verbosecall_enter(jd);
3169 #endif
3170
3171         /* get function address (this must happen before the stackframeinfo) */
3172
3173 #if !defined(WITH_STATIC_CLASSPATH)
3174         if (f == NULL)
3175                 codegen_add_patch_ref(cd, PATCHER_resolve_native, m, 0);
3176 #endif
3177
3178         M_MOV_IMM(f, REG_ITMP3);
3179
3180 #if defined(ENABLE_GC_CACAO)
3181         /* Save callee saved integer registers in stackframeinfo (GC may
3182            need to recover them during a collection). */
3183
3184         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3185                 OFFSET(stackframeinfo, intregs);
3186
3187         for (i = 0; i < INT_SAV_CNT; i++)
3188                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3189 #endif
3190
3191         /* save integer and float argument registers */
3192
3193         for (i = 0; i < md->paramcount; i++) {
3194                 if (!md->params[i].inmemory) {
3195                         s1 = md->params[i].regoff;
3196
3197                         switch (md->paramtypes[i].type) {
3198                         case TYPE_INT:
3199                         case TYPE_LNG:
3200                         case TYPE_ADR:
3201                                 M_LST(s1, REG_SP, i * 8);
3202                                 break;
3203                         case TYPE_FLT:
3204                         case TYPE_DBL:
3205                                 M_DST(s1, REG_SP, i * 8);
3206                                 break;
3207                         }
3208                 }
3209         }
3210
3211         M_AST(REG_ITMP3, REG_SP, md->paramcount * 8);
3212
3213         /* create dynamic stack info */
3214
3215         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3216         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3217         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3218         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3219         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3220         M_CALL(REG_ITMP1);
3221
3222         /* restore integer and float argument registers */
3223
3224         for (i = 0; i < md->paramcount; i++) {
3225                 if (!md->params[i].inmemory) {
3226                         s1 = md->params[i].regoff;
3227
3228                         switch (md->paramtypes[i].type) {
3229                         case TYPE_INT:
3230                         case TYPE_LNG:
3231                         case TYPE_ADR:
3232                                 M_LLD(s1, REG_SP, i * 8);
3233                                 break;
3234                         case TYPE_FLT:
3235                         case TYPE_DBL:
3236                                 M_DLD(s1, REG_SP, i * 8);
3237                                 break;
3238                         }
3239                 }
3240         }
3241
3242         M_ALD(REG_ITMP3, REG_SP, md->paramcount * 8);
3243
3244         /* copy or spill arguments to new locations */
3245
3246         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3247                 t  = md->paramtypes[i].type;
3248                 s2 = nmd->params[j].regoff;
3249
3250                 if (IS_INT_LNG_TYPE(t)) {
3251                         if (!md->params[i].inmemory) {
3252                                 s1 = md->params[i].regoff;
3253
3254                                 if (!nmd->params[j].inmemory)
3255                                         M_INTMOVE(s1, s2);
3256                                 else
3257                                         M_LST(s1, REG_SP, s2 * 8);
3258                         }
3259                         else {
3260                                 s1 = md->params[i].regoff + cd->stackframesize + 1;/* +1 (RA) */
3261                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
3262                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
3263                         }
3264                 }
3265                 else {
3266                         /* We only copy spilled float arguments, as the float
3267                            argument registers keep unchanged. */
3268
3269                         if (md->params[i].inmemory) {
3270                                 s1 = md->params[i].regoff + cd->stackframesize + 1;/* +1 (RA) */
3271
3272                                 if (IS_2_WORD_TYPE(t)) {
3273                                         M_DLD(REG_FTMP1, REG_SP, s1 * 8);
3274                                         M_DST(REG_FTMP1, REG_SP, s2 * 8);
3275                                 }
3276                                 else {
3277                                         M_FLD(REG_FTMP1, REG_SP, s1 * 8);
3278                                         M_FST(REG_FTMP1, REG_SP, s2 * 8);
3279                                 }
3280                         }
3281                 }
3282         }
3283
3284         /* put class into second argument register */
3285
3286         if (m->flags & ACC_STATIC)
3287                 M_MOV_IMM(m->class, REG_A1);
3288
3289         /* put env into first argument register */
3290
3291         M_MOV_IMM(_Jv_env, REG_A0);
3292
3293         /* do the native function call */
3294
3295         M_CALL(REG_ITMP3);
3296
3297         /* save return value */
3298
3299         switch (md->returntype.type) {
3300         case TYPE_INT:
3301         case TYPE_LNG:
3302         case TYPE_ADR:
3303                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3304                 break;
3305         case TYPE_FLT:
3306         case TYPE_DBL:
3307                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3308                 break;
3309         case TYPE_VOID:
3310                 break;
3311         }
3312
3313 #if !defined(NDEBUG)
3314         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3315                 emit_verbosecall_exit(jd);
3316 #endif
3317
3318         /* remove native stackframe info */
3319
3320         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3321         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3322         M_CALL(REG_ITMP1);
3323         M_MOV(REG_RESULT, REG_ITMP3);
3324
3325         /* restore return value */
3326
3327         switch (md->returntype.type) {
3328         case TYPE_INT:
3329         case TYPE_LNG:
3330         case TYPE_ADR:
3331                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3332                 break;
3333         case TYPE_FLT:
3334         case TYPE_DBL:
3335                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3336                 break;
3337         case TYPE_VOID:
3338                 break;
3339         }
3340
3341 #if defined(ENABLE_GC_CACAO)
3342         /* Restore callee saved integer registers from stackframeinfo (GC
3343            might have modified them during a collection). */
3344
3345         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3346                 OFFSET(stackframeinfo, intregs);
3347
3348         for (i = 0; i < INT_SAV_CNT; i++)
3349                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3350 #endif
3351
3352         /* remove stackframe */
3353
3354         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3355
3356         /* test for exception */
3357
3358         M_TEST(REG_ITMP3);
3359         M_BNE(1);
3360         M_RET;
3361
3362         /* handle exception */
3363
3364         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3365         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3366         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3367
3368         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3369         M_JMP(REG_ITMP3);
3370
3371         /* generate patcher stubs */
3372
3373         emit_patcher_stubs(jd);
3374 }
3375
3376
3377 /*
3378  * These are local overrides for various environment variables in Emacs.
3379  * Please do not remove this and leave it at the end of the file, where
3380  * Emacs will automagically detect them.
3381  * ---------------------------------------------------------------------
3382  * Local variables:
3383  * mode: c
3384  * indent-tabs-mode: t
3385  * c-basic-offset: 4
3386  * tab-width: 4
3387  * End:
3388  * vim:noexpandtab:sw=4:ts=4:
3389  */