src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    $Id: codegen.c 7702 2007-04-15 01:15:59Z michi $
  26
  27 */
  28
  29
  30 #include "config.h"
  31
  32 #include <assert.h>
  33 #include <stdio.h>
  34
  35 #include "vm/types.h"
  36
  37 #include "md-abi.h"
  38
  39 #include "vm/jit/x86_64/arch.h"
  40 #include "vm/jit/x86_64/codegen.h"
  41 #include "vm/jit/x86_64/emit.h"
  42
  43 #include "mm/memory.h"
  44 #include "native/jni.h"
  45 #include "native/native.h"
  46
  47 #if defined(ENABLE_THREADS)
  48 # include "threads/native/lock.h"
  49 #endif
  50
  51 #include "vm/builtin.h"
  52 #include "vm/exceptions.h"
  53 #include "vm/global.h"
  54 #include "vm/stringlocal.h"
  55 #include "vm/vm.h"
  56
  57 #include "vm/jit/abi.h"
  58 #include "vm/jit/asmpart.h"
  59 #include "vm/jit/codegen-common.h"
  60 #include "vm/jit/dseg.h"
  61 #include "vm/jit/emit-common.h"
  62 #include "vm/jit/jit.h"
  63 #include "vm/jit/methodheader.h"
  64 #include "vm/jit/parse.h"
  65 #include "vm/jit/patcher.h"
  66 #include "vm/jit/reg.h"
  67 #include "vm/jit/replace.h"
  68 #include "vm/jit/stacktrace.h"
  69
  70 #if defined(ENABLE_LSRA)
  71 # include "vm/jit/allocator/lsra.h"
  72 #endif
  73
  74 #include "vmcore/loader.h"
  75 #include "vmcore/options.h"
  76 #include "vmcore/statistics.h"
  77
  78
  79 /* codegen_emit ****************************************************************
  80
  81    Generates machine code.
  82
  83 *******************************************************************************/
  84
  85 bool codegen_emit(jitdata *jd)
  86 {
  87         methodinfo         *m;
  88         codeinfo           *code;
  89         codegendata        *cd;
  90         registerdata       *rd;
  91         s4                  len, s1, s2, s3, d, disp;
  92         u2                  currentline;
  93         ptrint              a;
  94         varinfo            *var, *dst;
  95         basicblock         *bptr;
  96         instruction        *iptr;
  97         exception_entry    *ex;
  98         constant_classref  *cr;
  99         unresolved_class   *uc;
 100         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 101         unresolved_method  *um;
 102         builtintable_entry *bte;
 103         methoddesc         *md;
 104         fieldinfo          *fi;
 105         unresolved_field   *uf;
 106         s4                  fieldtype;
 107         s4                 varindex;
 108
 109         /* get required compiler data */
 110
 111         m    = jd->m;
 112         code = jd->code;
 113         cd   = jd->cd;
 114         rd   = jd->rd;
 115
 116         /* prevent compiler warnings */
 117
 118         d   = 0;
 119         lm  = NULL;
 120         um  = NULL;
 121         bte = NULL;
 122
 123         {
 124         s4 i, p, t, l;
 125         s4 savedregs_num;
 126
 127         savedregs_num = 0;
 128
 129         /* space to save used callee saved registers */
 130
 131         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 132         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 133
 134         cd->stackframesize = rd->memuse + savedregs_num;
 135
 136 #if defined(ENABLE_THREADS)
 137         /* space to save argument of monitor_enter */
 138
 139         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 140                 cd->stackframesize++;
 141 #endif
 142
 143         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 144            native code e.g. libc or jni (alignment problems with
 145            movaps). */
 146
 147         if (!jd->isleafmethod || opt_verbosecall)
 148                 cd->stackframesize |= 0x1;
 149
 150         /* create method header */
 151
 152         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 153         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 154
 155 #if defined(ENABLE_THREADS)
 156         /* IsSync contains the offset relative to the stack pointer for the
 157            argument of monitor_exit used in the exception handler. Since the
 158            offset could be zero and give a wrong meaning of the flag it is
 159            offset by one.
 160         */
 161
 162         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 163                 (void) dseg_add_unique_s4(cd, (rd->memuse + 1) * 8); /* IsSync        */
 164         else
 165 #endif
 166                 (void) dseg_add_unique_s4(cd, 0);                  /* IsSync          */
 167
 168         (void) dseg_add_unique_s4(cd, jd->isleafmethod);               /* IsLeaf  */
 169         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 170         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 171
 172         (void) dseg_addlinenumbertablesize(cd);
 173
 174         (void) dseg_add_unique_s4(cd, jd->exceptiontablelength); /* ExTableSize   */
 175
 176         /* create exception table */
 177
 178         for (ex = jd->exceptiontable; ex != NULL; ex = ex->down) {
 179                 dseg_add_target(cd, ex->start);
 180                 dseg_add_target(cd, ex->end);
 181                 dseg_add_target(cd, ex->handler);
 182                 (void) dseg_add_unique_address(cd, ex->catchtype.any);
 183         }
 184
 185 #if defined(ENABLE_PROFILING)
 186         /* generate method profiling code */
 187
 188         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 189                 /* count frequency */
 190
 191                 M_MOV_IMM(code, REG_ITMP3);
 192                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 193
 194                 PROFILE_CYCLE_START;
 195         }
 196 #endif
 197
 198         /* create stack frame (if necessary) */
 199
 200         if (cd->stackframesize)
 201                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 202
 203         /* save used callee saved registers */
 204
 205         p = cd->stackframesize;
 206         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 207                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 208         }
 209         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 210                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 211         }
 212
 213         /* take arguments out of register or stack frame */
 214
 215         md = m->parseddesc;
 216
 217         for (p = 0, l = 0; p < md->paramcount; p++) {
 218                 t = md->paramtypes[p].type;
 219
 220                 varindex = jd->local_map[l * 5 + t];
 221
 222                 l++;
 223                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 224                         l++;
 225
 226                 if (varindex == UNUSED)
 227                         continue;
 228
 229                 var = VAR(varindex);
 230
 231                 s1 = md->params[p].regoff;
 232
 233                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 234                         s2 = rd->argintregs[s1];
 235                         if (!md->params[p].inmemory) {           /* register arguments    */
 236                                 if (!IS_INMEMORY(var->flags)) {      /* reg arg -> register   */
 237                                         M_INTMOVE(s2, var->vv.regoff);
 238
 239                                 } else {                             /* reg arg -> spilled    */
 240                                     M_LST(s2, REG_SP, var->vv.regoff * 8);
 241                                 }
 242
 243                         } else {                                 /* stack arguments       */
 244                                 if (!IS_INMEMORY(var->flags)) {      /* stack arg -> register */
 245                                         /* + 8 for return address */
 246                                         M_LLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 247
 248                                 } else {                             /* stack arg -> spilled  */
 249                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 250                                 }
 251                         }
 252
 253                 } else {                                     /* floating args         */
 254                         if (!md->params[p].inmemory) {           /* register arguments    */
 255                                 s2 = rd->argfltregs[s1];
 256                                 if (!IS_INMEMORY(var->flags)) {      /* reg arg -> register   */
 257                                         M_FLTMOVE(s2, var->vv.regoff);
 258
 259                                 } else {                                         /* reg arg -> spilled    */
 260                                         M_DST(s2, REG_SP, var->vv.regoff * 8);
 261                                 }
 262
 263                         } else {                                 /* stack arguments       */
 264                                 if (!IS_INMEMORY(var->flags)) {      /* stack-arg -> register */
 265                                         M_DLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 266
 267                                 } else {
 268                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 269                                 }
 270                         }
 271                 }
 272         }  /* end for */
 273
 274         /* save monitorenter argument */
 275
 276 #if defined(ENABLE_THREADS)
 277         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 278                 /* stack offset for monitor argument */
 279
 280                 s1 = rd->memuse;
 281
 282                 if (opt_verbosecall) {
 283                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 284
 285                         for (p = 0; p < INT_ARG_CNT; p++)
 286                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 287
 288                         for (p = 0; p < FLT_ARG_CNT; p++)
 289                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 290
 291                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 292                 }
 293
 294                 /* decide which monitor enter function to call */
 295
 296                 if (m->flags & ACC_STATIC) {
 297                         M_MOV_IMM(&m->class->object.header, REG_A0);
 298                 }
 299                 else {
 300                         M_TEST(REG_A0);
 301                         M_BNE(8);
 302                         M_ALD_MEM(REG_A0, EXCEPTION_HARDWARE_NULLPOINTER);
 303                 }
 304
 305                 M_AST(REG_A0, REG_SP, s1 * 8);
 306                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 307                 M_CALL(REG_ITMP1);
 308
 309                 if (opt_verbosecall) {
 310                         for (p = 0; p < INT_ARG_CNT; p++)
 311                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 312
 313                         for (p = 0; p < FLT_ARG_CNT; p++)
 314                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 315
 316                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 317                 }
 318         }
 319 #endif
 320
 321 #if !defined(NDEBUG)
 322         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 323                 emit_verbosecall_enter(jd);
 324 #endif /* !defined(NDEBUG) */
 325
 326         }
 327
 328         /* end of header generation */
 329
 330         /* create replacement points */
 331
 332         REPLACEMENT_POINTS_INIT(cd, jd);
 333
 334         /* walk through all basic blocks */
 335
 336         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 337
 338                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 339
 340                 if (bptr->flags >= BBREACHED) {
 341
 342                 /* branch resolving */
 343
 344                 codegen_resolve_branchrefs(cd, bptr);
 345
 346                 /* handle replacement points */
 347
 348                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 349
 350                 /* copy interface registers to their destination */
 351
 352                 len = bptr->indepth;
 353                 MCODECHECK(512);
 354
 355 #if defined(ENABLE_PROFILING)
 356                 /* generate basicblock profiling code */
 357
 358                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 359                         /* count frequency */
 360
 361                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 362                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 363
 364                         /* if this is an exception handler, start profiling again */
 365
 366                         if (bptr->type == BBTYPE_EXH)
 367                                 PROFILE_CYCLE_START;
 368                 }
 369 #endif
 370
 371 #if defined(ENABLE_LSRA)
 372                 if (opt_lsra) {
 373                         while (len) {
 374                                 len--;
 375                                 src = bptr->invars[len];
 376                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 377                                         if (bptr->type == BBTYPE_EXH) {
 378 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 379                                                 if (!IS_INMEMORY(src->flags))
 380                                                         d= src->vv.regoff;
 381                                                 else
 382                                                         d=REG_ITMP1;
 383                                                 M_INTMOVE(REG_ITMP1, d);
 384                                                 emit_store(jd, NULL, src, d);
 385                                         }
 386                                 }
 387                         }
 388
 389                 } else {
 390 #endif
 391
 392                 while (len) {
 393                         len--;
 394                         var = VAR(bptr->invars[len]);
 395                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 396                                 if (bptr->type == BBTYPE_EXH) {
 397                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 398                                         M_INTMOVE(REG_ITMP1, d);
 399                                         emit_store(jd, NULL, var, d);
 400                                 }
 401                         }
 402                         else {
 403                                 assert((var->flags & INOUT));
 404                         }
 405                 }
 406 #if defined(ENABLE_LSRA)
 407                 }
 408 #endif
 409                 /* walk through all instructions */
 410
 411                 len = bptr->icount;
 412                 currentline = 0;
 413
 414                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 415                         if (iptr->line != currentline) {
 416                                 dseg_addlinenumber(cd, iptr->line);
 417                                 currentline = iptr->line;
 418                         }
 419
 420                         MCODECHECK(1024);                         /* 1KB should be enough */
 421
 422                 switch (iptr->opc) {
 423                 case ICMD_NOP:        /* ...  ==> ...                                 */
 424                 case ICMD_POP:        /* ..., value  ==> ...                          */
 425                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 426                         break;
 427
 428                 case ICMD_INLINE_START:
 429
 430                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 431                         break;
 432
 433                 case ICMD_INLINE_BODY:
 434
 435                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 436                         dseg_addlinenumber_inline_start(cd, iptr);
 437                         dseg_addlinenumber(cd, iptr->line);
 438                         break;
 439
 440                 case ICMD_INLINE_END:
 441
 442                         dseg_addlinenumber_inline_end(cd, iptr);
 443                         dseg_addlinenumber(cd, iptr->line);
 444                         break;
 445
 446                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 447
 448                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 449                         emit_nullpointer_check(cd, iptr, s1);
 450                         break;
 451
 452                 /* constant operations ************************************************/
 453
 454                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 455
 456                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 457                         ICONST(d, iptr->sx.val.i);
 458                         emit_store_dst(jd, iptr, d);
 459                         break;
 460
 461                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 462
 463                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 464                         LCONST(d, iptr->sx.val.l);
 465                         emit_store_dst(jd, iptr, d);
 466                         break;
 467
 468                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 469
 470                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 471                         disp = dseg_add_float(cd, iptr->sx.val.f);
 472                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 473                         emit_store_dst(jd, iptr, d);
 474                         break;
 475
 476                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 477
 478                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 479                         disp = dseg_add_double(cd, iptr->sx.val.d);
 480                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 481                         emit_store_dst(jd, iptr, d);
 482                         break;
 483
 484                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 485
 486                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 487
 488                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 489                                 cr = iptr->sx.val.c.ref;
 490
 491 /*                              PROFILE_CYCLE_STOP; */
 492
 493                                 codegen_add_patch_ref(cd, PATCHER_aconst, cr, 0);
 494
 495 /*                              PROFILE_CYCLE_START; */
 496
 497                                 M_MOV_IMM(NULL, d);
 498
 499                         } else {
 500                                 if (iptr->sx.val.anyptr == 0)
 501                                         M_CLR(d);
 502                                 else
 503                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 504                         }
 505                         emit_store_dst(jd, iptr, d);
 506                         break;
 507
 508
 509                 /* load/store/copy/move operations ************************************/
 510
 511                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 512                 case ICMD_ALOAD:      /* s1 = local variable                          */
 513                 case ICMD_LLOAD:
 514                 case ICMD_FLOAD:
 515                 case ICMD_DLOAD:
 516                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 517                 case ICMD_LSTORE:
 518                 case ICMD_FSTORE:
 519                 case ICMD_DSTORE:
 520                 case ICMD_COPY:
 521                 case ICMD_MOVE:
 522
 523                         emit_copy(jd, iptr, VAROP(iptr->s1), VAROP(iptr->dst));
 524                         break;
 525
 526                 case ICMD_ASTORE:
 527                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 528                                 emit_copy(jd, iptr, VAROP(iptr->s1), VAROP(iptr->dst));
 529                         break;
 530
 531                 /* integer operations *************************************************/
 532
 533                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 534
 535                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 536                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 537                         M_INTMOVE(s1, d);
 538                         M_INEG(d);
 539                         emit_store_dst(jd, iptr, d);
 540                         break;
 541
 542                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 543
 544                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 545                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 546                         M_INTMOVE(s1, d);
 547                         M_LNEG(d);
 548                         emit_store_dst(jd, iptr, d);
 549                         break;
 550
 551                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 552
 553                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 554                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 555                         M_ISEXT(s1, d);
 556                         emit_store_dst(jd, iptr, d);
 557                         break;
 558
 559                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 560
 561                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 562                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 563                         M_IMOV(s1, d);
 564                         emit_store_dst(jd, iptr, d);
 565                         break;
 566
 567                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 568
 569                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 570                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 571                         M_BSEXT(s1, d);
 572                         emit_store_dst(jd, iptr, d);
 573                         break;
 574
 575                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 576
 577                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 578                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 579                         M_CZEXT(s1, d);
 580                         emit_store_dst(jd, iptr, d);
 581                         break;
 582
 583                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 584
 585                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 586                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 587                         M_SSEXT(s1, d);
 588                         emit_store_dst(jd, iptr, d);
 589                         break;
 590
 591
 592                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 593
 594                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 595                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 596                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 597                         if (s2 == d)
 598                                 M_IADD(s1, d);
 599                         else {
 600                                 M_INTMOVE(s1, d);
 601                                 M_IADD(s2, d);
 602                         }
 603                         emit_store_dst(jd, iptr, d);
 604                         break;
 605
 606                 case ICMD_IINC:
 607                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 608                                       /* sx.val.i = constant                             */
 609
 610                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 611                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 612
 613                         /* Using inc and dec is not faster than add (tested with
 614                            sieve). */
 615
 616                         M_INTMOVE(s1, d);
 617                         M_IADD_IMM(iptr->sx.val.i, d);
 618                         emit_store_dst(jd, iptr, d);
 619                         break;
 620
 621                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 622
 623                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 624                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 625                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 626                         if (s2 == d)
 627                                 M_LADD(s1, d);
 628                         else {
 629                                 M_INTMOVE(s1, d);
 630                                 M_LADD(s2, d);
 631                         }
 632                         emit_store_dst(jd, iptr, d);
 633                         break;
 634
 635                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 636                                       /* sx.val.l = constant                             */
 637
 638                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 639                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 640                         M_INTMOVE(s1, d);
 641                         if (IS_IMM32(iptr->sx.val.l))
 642                                 M_LADD_IMM(iptr->sx.val.l, d);
 643                         else {
 644                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 645                                 M_LADD(REG_ITMP2, d);
 646                         }
 647                         emit_store_dst(jd, iptr, d);
 648                         break;
 649
 650                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 651
 652                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 653                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 654                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 655                         if (s2 == d) {
 656                                 M_INTMOVE(s1, REG_ITMP1);
 657                                 M_ISUB(s2, REG_ITMP1);
 658                                 M_INTMOVE(REG_ITMP1, d);
 659                         } else {
 660                                 M_INTMOVE(s1, d);
 661                                 M_ISUB(s2, d);
 662                         }
 663                         emit_store_dst(jd, iptr, d);
 664                         break;
 665
 666                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 667                                       /* sx.val.i = constant                             */
 668
 669                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 670                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 671                         M_INTMOVE(s1, d);
 672                         M_ISUB_IMM(iptr->sx.val.i, d);
 673                         emit_store_dst(jd, iptr, d);
 674                         break;
 675
 676                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 677
 678                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 679                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 680                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 681                         if (s2 == d) {
 682                                 M_INTMOVE(s1, REG_ITMP1);
 683                                 M_LSUB(s2, REG_ITMP1);
 684                                 M_INTMOVE(REG_ITMP1, d);
 685                         } else {
 686                                 M_INTMOVE(s1, d);
 687                                 M_LSUB(s2, d);
 688                         }
 689                         emit_store_dst(jd, iptr, d);
 690                         break;
 691
 692                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 693                                       /* sx.val.l = constant                             */
 694
 695                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 696                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 697                         M_INTMOVE(s1, d);
 698                         if (IS_IMM32(iptr->sx.val.l))
 699                                 M_LSUB_IMM(iptr->sx.val.l, d);
 700                         else {
 701                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 702                                 M_LSUB(REG_ITMP2, d);
 703                         }
 704                         emit_store_dst(jd, iptr, d);
 705                         break;
 706
 707                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 708
 709                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 710                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 711                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 712                         if (s2 == d)
 713                                 M_IMUL(s1, d);
 714                         else {
 715                                 M_INTMOVE(s1, d);
 716                                 M_IMUL(s2, d);
 717                         }
 718                         emit_store_dst(jd, iptr, d);
 719                         break;
 720
 721                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 722                                       /* sx.val.i = constant                             */
 723
 724                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 725                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 726                         if (iptr->sx.val.i == 2) {
 727                                 M_INTMOVE(s1, d);
 728                                 M_ISLL_IMM(1, d);
 729                         } else
 730                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 731                         emit_store_dst(jd, iptr, d);
 732                         break;
 733
 734                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 735
 736                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 737                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 738                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 739                         if (s2 == d)
 740                                 M_LMUL(s1, d);
 741                         else {
 742                                 M_INTMOVE(s1, d);
 743                                 M_LMUL(s2, d);
 744                         }
 745                         emit_store_dst(jd, iptr, d);
 746                         break;
 747
 748                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 749                                       /* sx.val.l = constant                             */
 750
 751                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 752                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 753                         if (IS_IMM32(iptr->sx.val.l))
 754                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 755                         else {
 756                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 757                                 M_INTMOVE(s1, d);
 758                                 M_LMUL(REG_ITMP2, d);
 759                         }
 760                         emit_store_dst(jd, iptr, d);
 761                         break;
 762
 763                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 764
 765                         s1 = emit_load_s1(jd, iptr, RAX);
 766                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 767                         d = codegen_reg_of_dst(jd, iptr, RAX);
 768
 769                         M_INTMOVE(s1, RAX);
 770                         M_INTMOVE(s2, REG_ITMP3);
 771                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 772
 773                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 774
 775                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 776                         M_BNE(4 + 6);
 777                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 778                         M_BEQ(1 + 3);                                          /* 6 bytes */
 779
 780                         emit_cltd(cd);                                         /* 1 byte  */
 781                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 782
 783                         M_INTMOVE(RAX, d);
 784                         emit_store_dst(jd, iptr, d);
 785                         dst = VAROP(iptr->dst);
 786                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 787                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 788                         break;
 789
 790                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 791
 792                         s1 = emit_load_s1(jd, iptr, RAX);
 793                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 794                         d = codegen_reg_of_dst(jd, iptr, RDX);
 795
 796                         M_INTMOVE(s1, RAX);
 797                         M_INTMOVE(s2, REG_ITMP3);
 798                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 799
 800                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 801
 802                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 803                         M_BNE(3 + 4 + 6);
 804                         M_CLR(RDX);                                            /* 3 bytes */
 805                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 806                         M_BEQ(1 + 3);                                          /* 6 bytes */
 807
 808                         emit_cltd(cd);                                         /* 1 byte  */
 809                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 810
 811                         M_INTMOVE(RDX, d);
 812                         emit_store_dst(jd, iptr, d);
 813                         dst = VAROP(iptr->dst);
 814                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 815                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 816                         break;
 817
 818                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 819                                       /* sx.val.i = constant                             */
 820
 821                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 822                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 823                         M_INTMOVE(s1, REG_ITMP1);
 824                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 825                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 826                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 827                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 828                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 829                         emit_store_dst(jd, iptr, d);
 830                         break;
 831
 832                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 833                                       /* sx.val.i = constant                             */
 834
 835                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 836                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 837                         M_INTMOVE(s1, REG_ITMP1);
 838                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 839                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 840                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 841                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 842                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 843                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 844                         emit_store_dst(jd, iptr, d);
 845                         break;
 846
 847
 848                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 849
 850                         s1 = emit_load_s1(jd, iptr, RAX);
 851                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 852                         d = codegen_reg_of_dst(jd, iptr, RAX);
 853
 854                         M_INTMOVE(s1, RAX);
 855                         M_INTMOVE(s2, REG_ITMP3);
 856                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 857
 858                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 859
 860                         /* check as described in jvm spec */
 861                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 862                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 863                         M_BNE(4 + 6);
 864                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 865                         M_BEQ(2 + 3);                                          /* 6 bytes */
 866
 867                         emit_cqto(cd);                                         /* 2 bytes */
 868                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 869
 870                         M_INTMOVE(RAX, d);
 871                         emit_store_dst(jd, iptr, d);
 872                         dst = VAROP(iptr->dst);
 873                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 874                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 875                         break;
 876
 877                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 878
 879                         s1 = emit_load_s1(jd, iptr, RAX);
 880                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 881                         d = codegen_reg_of_dst(jd, iptr, RDX);
 882
 883                         M_INTMOVE(s1, RAX);
 884                         M_INTMOVE(s2, REG_ITMP3);
 885                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 886
 887                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 888
 889                         /* check as described in jvm spec */
 890                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 891                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 892                         M_BNE(3 + 4 + 6);
 893                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 894                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 895                         M_BEQ(2 + 3);                                          /* 6 bytes */
 896
 897                         emit_cqto(cd);                                         /* 2 bytes */
 898                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 899
 900                         M_INTMOVE(RDX, d);
 901                         emit_store_dst(jd, iptr, d);
 902                         dst = VAROP(iptr->dst);
 903                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 904                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 905                         break;
 906
 907                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 908                                       /* sx.val.i = constant                             */
 909
 910                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 911                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 912                         M_INTMOVE(s1, REG_ITMP1);
 913                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 914                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 915                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 916                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 917                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 918                         emit_store_dst(jd, iptr, d);
 919                         break;
 920
 921                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 922                                       /* sx.val.l = constant                             */
 923
 924                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 925                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 926                         M_INTMOVE(s1, REG_ITMP1);
 927                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 928                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 929                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 930                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 931                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 932                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 933                         emit_store_dst(jd, iptr, d);
 934                         break;
 935
 936                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 937
 938                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 939                         emit_ishift(jd, SHIFT_SHL, iptr);
 940                         break;
 941
 942                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 943                                       /* sx.val.i = constant                             */
 944
 945                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 946                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 947                         M_INTMOVE(s1, d);
 948                         M_ISLL_IMM(iptr->sx.val.i, d);
 949                         emit_store_dst(jd, iptr, d);
 950                         break;
 951
 952                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 953
 954                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 955                         emit_ishift(jd, SHIFT_SAR, iptr);
 956                         break;
 957
 958                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 959                                       /* sx.val.i = constant                             */
 960
 961                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 962                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 963                         M_INTMOVE(s1, d);
 964                         M_ISRA_IMM(iptr->sx.val.i, d);
 965                         emit_store_dst(jd, iptr, d);
 966                         break;
 967
 968                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 969
 970                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 971                         emit_ishift(jd, SHIFT_SHR, iptr);
 972                         break;
 973
 974                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 975                                       /* sx.val.i = constant                             */
 976
 977                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 978                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 979                         M_INTMOVE(s1, d);
 980                         M_ISRL_IMM(iptr->sx.val.i, d);
 981                         emit_store_dst(jd, iptr, d);
 982                         break;
 983
 984                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 985
 986                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 987                         emit_lshift(jd, SHIFT_SHL, iptr);
 988                         break;
 989
 990         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 991                                           /* sx.val.i = constant                             */
 992
 993                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 994                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 995                         M_INTMOVE(s1, d);
 996                         M_LSLL_IMM(iptr->sx.val.i, d);
 997                         emit_store_dst(jd, iptr, d);
 998                         break;
 999
1000                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1001
1002                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1003                         emit_lshift(jd, SHIFT_SAR, iptr);
1004                         break;
1005
1006                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1007                                       /* sx.val.i = constant                             */
1008
1009                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1010                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1011                         M_INTMOVE(s1, d);
1012                         M_LSRA_IMM(iptr->sx.val.i, d);
1013                         emit_store_dst(jd, iptr, d);
1014                         break;
1015
1016                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1017
1018                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1019                         emit_lshift(jd, SHIFT_SHR, iptr);
1020                         break;
1021
1022                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1023                                       /* sx.val.l = constant                             */
1024
1025                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1026                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1027                         M_INTMOVE(s1, d);
1028                         M_LSRL_IMM(iptr->sx.val.i, d);
1029                         emit_store_dst(jd, iptr, d);
1030                         break;
1031
1032                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1033
1034                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1035                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1036                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1037                         if (s2 == d)
1038                                 M_IAND(s1, d);
1039                         else {
1040                                 M_INTMOVE(s1, d);
1041                                 M_IAND(s2, d);
1042                         }
1043                         emit_store_dst(jd, iptr, d);
1044                         break;
1045
1046                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1047                                       /* sx.val.i = constant                             */
1048
1049                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1050                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1051                         M_INTMOVE(s1, d);
1052                         M_IAND_IMM(iptr->sx.val.i, d);
1053                         emit_store_dst(jd, iptr, d);
1054                         break;
1055
1056                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1057
1058                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1059                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1060                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1061                         if (s2 == d)
1062                                 M_LAND(s1, d);
1063                         else {
1064                                 M_INTMOVE(s1, d);
1065                                 M_LAND(s2, d);
1066                         }
1067                         emit_store_dst(jd, iptr, d);
1068                         break;
1069
1070                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1071                                       /* sx.val.l = constant                             */
1072
1073                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1074                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1075                         M_INTMOVE(s1, d);
1076                         if (IS_IMM32(iptr->sx.val.l))
1077                                 M_LAND_IMM(iptr->sx.val.l, d);
1078                         else {
1079                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1080                                 M_LAND(REG_ITMP2, d);
1081                         }
1082                         emit_store_dst(jd, iptr, d);
1083                         break;
1084
1085                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1086
1087                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1088                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1089                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1090                         if (s2 == d)
1091                                 M_IOR(s1, d);
1092                         else {
1093                                 M_INTMOVE(s1, d);
1094                                 M_IOR(s2, d);
1095                         }
1096                         emit_store_dst(jd, iptr, d);
1097                         break;
1098
1099                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1100                                       /* sx.val.i = constant                             */
1101
1102                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1103                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1104                         M_INTMOVE(s1, d);
1105                         M_IOR_IMM(iptr->sx.val.i, d);
1106                         emit_store_dst(jd, iptr, d);
1107                         break;
1108
1109                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1110
1111                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1112                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1113                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1114                         if (s2 == d)
1115                                 M_LOR(s1, d);
1116                         else {
1117                                 M_INTMOVE(s1, d);
1118                                 M_LOR(s2, d);
1119                         }
1120                         emit_store_dst(jd, iptr, d);
1121                         break;
1122
1123                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1124                                       /* sx.val.l = constant                             */
1125
1126                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1127                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1128                         M_INTMOVE(s1, d);
1129                         if (IS_IMM32(iptr->sx.val.l))
1130                                 M_LOR_IMM(iptr->sx.val.l, d);
1131                         else {
1132                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1133                                 M_LOR(REG_ITMP2, d);
1134                         }
1135                         emit_store_dst(jd, iptr, d);
1136                         break;
1137
1138                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1139
1140                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1141                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1142                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1143                         if (s2 == d)
1144                                 M_IXOR(s1, d);
1145                         else {
1146                                 M_INTMOVE(s1, d);
1147                                 M_IXOR(s2, d);
1148                         }
1149                         emit_store_dst(jd, iptr, d);
1150                         break;
1151
1152                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1153                                       /* sx.val.i = constant                             */
1154
1155                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1156                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1157                         M_INTMOVE(s1, d);
1158                         M_IXOR_IMM(iptr->sx.val.i, d);
1159                         emit_store_dst(jd, iptr, d);
1160                         break;
1161
1162                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1163
1164                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1165                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1166                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1167                         if (s2 == d)
1168                                 M_LXOR(s1, d);
1169                         else {
1170                                 M_INTMOVE(s1, d);
1171                                 M_LXOR(s2, d);
1172                         }
1173                         emit_store_dst(jd, iptr, d);
1174                         break;
1175
1176                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1177                                       /* sx.val.l = constant                             */
1178
1179                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1180                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1181                         M_INTMOVE(s1, d);
1182                         if (IS_IMM32(iptr->sx.val.l))
1183                                 M_LXOR_IMM(iptr->sx.val.l, d);
1184                         else {
1185                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1186                                 M_LXOR(REG_ITMP2, d);
1187                         }
1188                         emit_store_dst(jd, iptr, d);
1189                         break;
1190
1191
1192                 /* floating operations ************************************************/
1193
1194                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1195
1196                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1197                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1198                         disp = dseg_add_s4(cd, 0x80000000);
1199                         M_FLTMOVE(s1, d);
1200                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1201                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1202                         emit_store_dst(jd, iptr, d);
1203                         break;
1204
1205                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1206
1207                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1208                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1209                         disp = dseg_add_s8(cd, 0x8000000000000000);
1210                         M_FLTMOVE(s1, d);
1211                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1212                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1213                         emit_store_dst(jd, iptr, d);
1214                         break;
1215
1216                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1217
1218                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1219                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1220                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1221                         if (s2 == d)
1222                                 M_FADD(s1, d);
1223                         else {
1224                                 M_FLTMOVE(s1, d);
1225                                 M_FADD(s2, d);
1226                         }
1227                         emit_store_dst(jd, iptr, d);
1228                         break;
1229
1230                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1231
1232                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1233                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1234                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1235                         if (s2 == d)
1236                                 M_DADD(s1, d);
1237                         else {
1238                                 M_FLTMOVE(s1, d);
1239                                 M_DADD(s2, d);
1240                         }
1241                         emit_store_dst(jd, iptr, d);
1242                         break;
1243
1244                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1245
1246                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1247                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1248                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1249                         if (s2 == d) {
1250                                 M_FLTMOVE(s2, REG_FTMP2);
1251                                 s2 = REG_FTMP2;
1252                         }
1253                         M_FLTMOVE(s1, d);
1254                         M_FSUB(s2, d);
1255                         emit_store_dst(jd, iptr, d);
1256                         break;
1257
1258                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1259
1260                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1261                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1262                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1263                         if (s2 == d) {
1264                                 M_FLTMOVE(s2, REG_FTMP2);
1265                                 s2 = REG_FTMP2;
1266                         }
1267                         M_FLTMOVE(s1, d);
1268                         M_DSUB(s2, d);
1269                         emit_store_dst(jd, iptr, d);
1270                         break;
1271
1272                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1273
1274                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1275                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1276                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1277                         if (s2 == d)
1278                                 M_FMUL(s1, d);
1279                         else {
1280                                 M_FLTMOVE(s1, d);
1281                                 M_FMUL(s2, d);
1282                         }
1283                         emit_store_dst(jd, iptr, d);
1284                         break;
1285
1286                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1287
1288                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1289                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1290                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1291                         if (s2 == d)
1292                                 M_DMUL(s1, d);
1293                         else {
1294                                 M_FLTMOVE(s1, d);
1295                                 M_DMUL(s2, d);
1296                         }
1297                         emit_store_dst(jd, iptr, d);
1298                         break;
1299
1300                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1301
1302                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1303                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1304                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1305                         if (s2 == d) {
1306                                 M_FLTMOVE(s2, REG_FTMP2);
1307                                 s2 = REG_FTMP2;
1308                         }
1309                         M_FLTMOVE(s1, d);
1310                         M_FDIV(s2, d);
1311                         emit_store_dst(jd, iptr, d);
1312                         break;
1313
1314                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1315
1316                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1317                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1318                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1319                         if (s2 == d) {
1320                                 M_FLTMOVE(s2, REG_FTMP2);
1321                                 s2 = REG_FTMP2;
1322                         }
1323                         M_FLTMOVE(s1, d);
1324                         M_DDIV(s2, d);
1325                         emit_store_dst(jd, iptr, d);
1326                         break;
1327
1328                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1329
1330                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1331                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1332                         M_CVTIF(s1, d);
1333                         emit_store_dst(jd, iptr, d);
1334                         break;
1335
1336                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1337
1338                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1339                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1340                         M_CVTID(s1, d);
1341                         emit_store_dst(jd, iptr, d);
1342                         break;
1343
1344                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1345
1346                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1347                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1348                         M_CVTLF(s1, d);
1349                         emit_store_dst(jd, iptr, d);
1350                         break;
1351
1352                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1353
1354                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1355                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1356                         M_CVTLD(s1, d);
1357                         emit_store_dst(jd, iptr, d);
1358                         break;
1359
1360                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1361
1362                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1363                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1364                         M_CVTFI(s1, d);
1365                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1366                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1367                                 ((REG_RESULT == d) ? 0 : 3);
1368                         M_BNE(disp);
1369                         M_FLTMOVE(s1, REG_FTMP1);
1370                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1371                         M_CALL(REG_ITMP2);
1372                         M_INTMOVE(REG_RESULT, d);
1373                         emit_store_dst(jd, iptr, d);
1374                         break;
1375
1376                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1377
1378                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1379                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1380                         M_CVTDI(s1, d);
1381                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1382                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1383                                 ((REG_RESULT == d) ? 0 : 3);
1384                         M_BNE(disp);
1385                         M_FLTMOVE(s1, REG_FTMP1);
1386                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1387                         M_CALL(REG_ITMP2);
1388                         M_INTMOVE(REG_RESULT, d);
1389                         emit_store_dst(jd, iptr, d);
1390                         break;
1391
1392                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1393
1394                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1395                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1396                         M_CVTFL(s1, d);
1397                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1398                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1399                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1400                                 ((REG_RESULT == d) ? 0 : 3);
1401                         M_BNE(disp);
1402                         M_FLTMOVE(s1, REG_FTMP1);
1403                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1404                         M_CALL(REG_ITMP2);
1405                         M_INTMOVE(REG_RESULT, d);
1406                         emit_store_dst(jd, iptr, d);
1407                         break;
1408
1409                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1410
1411                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1412                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1413                         M_CVTDL(s1, d);
1414                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1415                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1416                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1417                                 ((REG_RESULT == d) ? 0 : 3);
1418                         M_BNE(disp);
1419                         M_FLTMOVE(s1, REG_FTMP1);
1420                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1421                         M_CALL(REG_ITMP2);
1422                         M_INTMOVE(REG_RESULT, d);
1423                         emit_store_dst(jd, iptr, d);
1424                         break;
1425
1426                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1427
1428                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1429                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1430                         M_CVTFD(s1, d);
1431                         emit_store_dst(jd, iptr, d);
1432                         break;
1433
1434                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1435
1436                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1437                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1438                         M_CVTDF(s1, d);
1439                         emit_store_dst(jd, iptr, d);
1440                         break;
1441
1442                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1443                                           /* == => 0, < => 1, > => -1 */
1444
1445                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1446                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1447                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1448                         M_CLR(d);
1449                         M_MOV_IMM(1, REG_ITMP1);
1450                         M_MOV_IMM(-1, REG_ITMP2);
1451                         emit_ucomiss_reg_reg(cd, s1, s2);
1452                         M_CMOVULT(REG_ITMP1, d);
1453                         M_CMOVUGT(REG_ITMP2, d);
1454                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1455                         emit_store_dst(jd, iptr, d);
1456                         break;
1457
1458                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1459                                           /* == => 0, < => 1, > => -1 */
1460
1461                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1462                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1463                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1464                         M_CLR(d);
1465                         M_MOV_IMM(1, REG_ITMP1);
1466                         M_MOV_IMM(-1, REG_ITMP2);
1467                         emit_ucomiss_reg_reg(cd, s1, s2);
1468                         M_CMOVULT(REG_ITMP1, d);
1469                         M_CMOVUGT(REG_ITMP2, d);
1470                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1471                         emit_store_dst(jd, iptr, d);
1472                         break;
1473
1474                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1475                                           /* == => 0, < => 1, > => -1 */
1476
1477                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1478                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1479                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1480                         M_CLR(d);
1481                         M_MOV_IMM(1, REG_ITMP1);
1482                         M_MOV_IMM(-1, REG_ITMP2);
1483                         emit_ucomisd_reg_reg(cd, s1, s2);
1484                         M_CMOVULT(REG_ITMP1, d);
1485                         M_CMOVUGT(REG_ITMP2, d);
1486                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1487                         emit_store_dst(jd, iptr, d);
1488                         break;
1489
1490                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1491                                           /* == => 0, < => 1, > => -1 */
1492
1493                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1494                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1495                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1496                         M_CLR(d);
1497                         M_MOV_IMM(1, REG_ITMP1);
1498                         M_MOV_IMM(-1, REG_ITMP2);
1499                         emit_ucomisd_reg_reg(cd, s1, s2);
1500                         M_CMOVULT(REG_ITMP1, d);
1501                         M_CMOVUGT(REG_ITMP2, d);
1502                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1503                         emit_store_dst(jd, iptr, d);
1504                         break;
1505
1506
1507                 /* memory operations **************************************************/
1508
1509                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1510
1511                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1512                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1513                         /* implicit null-pointer check */
1514                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1515                         emit_store_dst(jd, iptr, d);
1516                         break;
1517
1518                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1519
1520                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1521                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1522                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1523                         /* implicit null-pointer check */
1524                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1525                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1526                         emit_store_dst(jd, iptr, d);
1527                         break;
1528
1529                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1530
1531                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1532                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1533                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1534                         /* implicit null-pointer check */
1535                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1536                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1537                         emit_store_dst(jd, iptr, d);
1538                         break;
1539
1540                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1541
1542                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1543                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1544                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1545                         /* implicit null-pointer check */
1546                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1547                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1548                         emit_store_dst(jd, iptr, d);
1549                         break;
1550
1551                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1552
1553                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1554                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1555                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1556                         /* implicit null-pointer check */
1557                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1558                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1559                         emit_store_dst(jd, iptr, d);
1560                         break;
1561
1562                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1563
1564                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1565                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1566                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1567                         /* implicit null-pointer check */
1568                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1569                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1570                         emit_store_dst(jd, iptr, d);
1571                         break;
1572
1573                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1574
1575                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1576                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1577                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1578                         /* implicit null-pointer check */
1579                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1580                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1581                         emit_store_dst(jd, iptr, d);
1582                         break;
1583
1584                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1585
1586                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1587                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1588                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1589                         /* implicit null-pointer check */
1590                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1591                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1592                         emit_store_dst(jd, iptr, d);
1593                         break;
1594
1595                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1596
1597                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1598                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1599                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1600                         /* implicit null-pointer check */
1601                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1602                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1603                         emit_store_dst(jd, iptr, d);
1604                         break;
1605
1606
1607                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1608
1609                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1610                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1611                         /* implicit null-pointer check */
1612                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1613                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1614                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1615                         break;
1616
1617                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1618
1619                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1620                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1621                         /* implicit null-pointer check */
1622                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1623                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1624                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1625                         break;
1626
1627                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1628
1629                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1630                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1631                         /* implicit null-pointer check */
1632                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1633                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1634                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1635                         break;
1636
1637                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1638
1639                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1640                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1641                         /* implicit null-pointer check */
1642                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1643                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1644                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
1645                         break;
1646
1647                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1648
1649                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1650                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1651                         /* implicit null-pointer check */
1652                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1653                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1654                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
1655                         break;
1656
1657                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1658
1659                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1660                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1661                         /* implicit null-pointer check */
1662                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1663                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1664                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
1665                         break;
1666
1667                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1668
1669                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1670                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1671                         /* implicit null-pointer check */
1672                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1673                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1674                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
1675                         break;
1676
1677                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1678
1679                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1680                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1681                         /* implicit null-pointer check */
1682                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1683                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1684
1685                         M_MOV(s1, REG_A0);
1686                         M_MOV(s3, REG_A1);
1687                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
1688                         M_CALL(REG_ITMP1);
1689                         emit_exception_check(cd, iptr);
1690
1691                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1692                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1693                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1694                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1695                         break;
1696
1697
1698                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1699
1700                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1701                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1702                         /* implicit null-pointer check */
1703                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1704                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1705                         break;
1706
1707                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1708
1709                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1710                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1711                         /* implicit null-pointer check */
1712                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1713                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray, data[0]), s1, s2, 1);
1714                         break;
1715
1716                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1717
1718                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1719                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1720                         /* implicit null-pointer check */
1721                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1722                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1723                         break;
1724
1725                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1726
1727                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1728                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1729                         /* implicit null-pointer check */
1730                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1731                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray, data[0]), s1, s2, 2);
1732                         break;
1733
1734                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1735
1736                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1737                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1738                         /* implicit null-pointer check */
1739                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1740
1741                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1742                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1743                         }
1744                         else {
1745                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1746                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
1747                         }
1748                         break;
1749
1750                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1751
1752                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1753                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1754                         /* implicit null-pointer check */
1755                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1756                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1757                         break;
1758
1759
1760                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1761
1762                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1763                                 uf        = iptr->sx.s23.s3.uf;
1764                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1765                                 disp      = dseg_add_unique_address(cd, NULL);
1766                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1767
1768                                 /* must be calculated before codegen_add_patch_ref */
1769
1770                                 if (opt_shownops)
1771                                         disp -= PATCHER_CALL_SIZE;
1772
1773 /*                              PROFILE_CYCLE_STOP; */
1774
1775                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1776
1777 /*                              PROFILE_CYCLE_START; */
1778                         }
1779                         else {
1780                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1781                                 fieldtype = fi->type;
1782                                 disp      = dseg_add_address(cd, &(fi->value));
1783                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1784
1785                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1786                                         PROFILE_CYCLE_STOP;
1787
1788                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1789
1790                                         PROFILE_CYCLE_START;
1791
1792                                         if (opt_shownops)
1793                                                 disp -= PATCHER_CALL_SIZE;
1794                                 }
1795                         }
1796
1797                         /* This approach is much faster than moving the field
1798                            address inline into a register. */
1799
1800                         M_ALD(REG_ITMP1, RIP, disp);
1801
1802                         switch (fieldtype) {
1803                         case TYPE_INT:
1804                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1805                                 M_ILD(d, REG_ITMP1, 0);
1806                                 break;
1807                         case TYPE_LNG:
1808                         case TYPE_ADR:
1809                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1810                                 M_LLD(d, REG_ITMP1, 0);
1811                                 break;
1812                         case TYPE_FLT:
1813                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1814                                 M_FLD(d, REG_ITMP1, 0);
1815                                 break;
1816                         case TYPE_DBL:
1817                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1818                                 M_DLD(d, REG_ITMP1, 0);
1819                                 break;
1820                         }
1821                         emit_store_dst(jd, iptr, d);
1822                         break;
1823
1824                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1825
1826                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1827                                 uf        = iptr->sx.s23.s3.uf;
1828                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1829                                 disp      = dseg_add_unique_address(cd, NULL);
1830                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1831
1832                                 /* must be calculated before codegen_add_patch_ref */
1833
1834                                 if (opt_shownops)
1835                                         disp -= PATCHER_CALL_SIZE;
1836
1837 /*                              PROFILE_CYCLE_STOP; */
1838
1839                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1840
1841 /*                              PROFILE_CYCLE_START; */
1842                         }
1843                         else {
1844                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1845                                 fieldtype = fi->type;
1846                                 disp      = dseg_add_address(cd, &(fi->value));
1847                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1848
1849                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1850                                         PROFILE_CYCLE_STOP;
1851
1852                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1853
1854                                         PROFILE_CYCLE_START;
1855
1856                                         if (opt_shownops)
1857                                                 disp -= PATCHER_CALL_SIZE;
1858                                 }
1859                         }
1860
1861                         /* This approach is much faster than moving the field
1862                            address inline into a register. */
1863
1864                         M_ALD(REG_ITMP1, RIP, disp);
1865
1866                         switch (fieldtype) {
1867                         case TYPE_INT:
1868                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1869                                 M_IST(s1, REG_ITMP1, 0);
1870                                 break;
1871                         case TYPE_LNG:
1872                         case TYPE_ADR:
1873                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1874                                 M_LST(s1, REG_ITMP1, 0);
1875                                 break;
1876                         case TYPE_FLT:
1877                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1878                                 M_FST(s1, REG_ITMP1, 0);
1879                                 break;
1880                         case TYPE_DBL:
1881                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1882                                 M_DST(s1, REG_ITMP1, 0);
1883                                 break;
1884                         }
1885                         break;
1886
1887                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1888                                           /* val = value (in current instruction)     */
1889                                           /* following NOP)                           */
1890
1891                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1892                                 uf        = iptr->sx.s23.s3.uf;
1893                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1894                                 disp      = dseg_add_unique_address(cd, NULL);
1895                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1896
1897                                 /* must be calculated before codegen_add_patch_ref */
1898
1899                                 if (opt_shownops)
1900                                         disp -= PATCHER_CALL_SIZE;
1901
1902 /*                              PROFILE_CYCLE_STOP; */
1903
1904                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1905
1906 /*                              PROFILE_CYCLE_START; */
1907                         }
1908                         else {
1909                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1910                                 fieldtype = fi->type;
1911                                 disp      = dseg_add_address(cd, &(fi->value));
1912                                 disp      = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
1913
1914                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1915                                         PROFILE_CYCLE_STOP;
1916
1917                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1918
1919                                         PROFILE_CYCLE_START;
1920
1921                                         if (opt_shownops)
1922                                                 disp -= PATCHER_CALL_SIZE;
1923                                 }
1924                         }
1925
1926                         /* This approach is much faster than moving the field
1927                            address inline into a register. */
1928
1929                         M_ALD(REG_ITMP1, RIP, disp);
1930
1931                         switch (fieldtype) {
1932                         case TYPE_INT:
1933                         case TYPE_FLT:
1934                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1935                                 break;
1936                         case TYPE_LNG:
1937                         case TYPE_ADR:
1938                         case TYPE_DBL:
1939                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1940                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1941                                 else {
1942                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1943                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1944                                 }
1945                                 break;
1946                         }
1947                         break;
1948
1949                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1950
1951                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1952
1953                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1954                                 uf        = iptr->sx.s23.s3.uf;
1955                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1956                                 disp      = 0;
1957
1958 /*                              PROFILE_CYCLE_STOP; */
1959
1960                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
1961
1962 /*                              PROFILE_CYCLE_START; */
1963                         }
1964                         else {
1965                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1966                                 fieldtype = fi->type;
1967                                 disp      = fi->offset;
1968                         }
1969
1970                         /* implicit null-pointer check */
1971                         switch (fieldtype) {
1972                         case TYPE_INT:
1973                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1974                                 M_ILD32(d, s1, disp);
1975                                 break;
1976                         case TYPE_LNG:
1977                         case TYPE_ADR:
1978                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1979                                 M_LLD32(d, s1, disp);
1980                                 break;
1981                         case TYPE_FLT:
1982                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1983                                 M_FLD32(d, s1, disp);
1984                                 break;
1985                         case TYPE_DBL:
1986                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1987                                 M_DLD32(d, s1, disp);
1988                                 break;
1989                         }
1990                         emit_store_dst(jd, iptr, d);
1991                         break;
1992
1993                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1994
1995                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1996                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1997
1998                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1999                                 uf        = iptr->sx.s23.s3.uf;
2000                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2001                                 disp      = 0;
2002
2003 /*                              PROFILE_CYCLE_STOP; */
2004
2005                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
2006
2007 /*                              PROFILE_CYCLE_START; */
2008                         }
2009                         else {
2010                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2011                                 fieldtype = fi->type;
2012                                 disp      = fi->offset;
2013                         }
2014
2015                         /* implicit null-pointer check */
2016                         switch (fieldtype) {
2017                         case TYPE_INT:
2018                                 M_IST32(s2, s1, disp);
2019                                 break;
2020                         case TYPE_LNG:
2021                         case TYPE_ADR:
2022                                 M_LST32(s2, s1, disp);
2023                                 break;
2024                         case TYPE_FLT:
2025                                 M_FST32(s2, s1, disp);
2026                                 break;
2027                         case TYPE_DBL:
2028                                 M_DST32(s2, s1, disp);
2029                                 break;
2030                         }
2031                         break;
2032
2033                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2034                                           /* val = value (in current instruction)     */
2035                                           /* following NOP)                           */
2036
2037                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2038
2039                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2040                                 uf        = iptr->sx.s23.s3.uf;
2041                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2042                                 disp      = 0;
2043
2044 /*                              PROFILE_CYCLE_STOP; */
2045
2046                                 codegen_add_patch_ref(cd, PATCHER_putfieldconst, uf, 0);
2047
2048 /*                              PROFILE_CYCLE_START; */
2049                         }
2050                         else {
2051                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2052                                 fieldtype = fi->type;
2053                                 disp      = fi->offset;
2054                         }
2055
2056                         /* implicit null-pointer check */
2057                         switch (fieldtype) {
2058                         case TYPE_INT:
2059                         case TYPE_FLT:
2060                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2061                                 break;
2062                         case TYPE_LNG:
2063                         case TYPE_ADR:
2064                         case TYPE_DBL:
2065                                 /* XXX why no check for IS_IMM32? */
2066                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2067                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2068                                 break;
2069                         }
2070                         break;
2071
2072
2073                 /* branch operations **************************************************/
2074
2075                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2076
2077                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2078                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2079
2080                         PROFILE_CYCLE_STOP;
2081
2082 #ifdef ENABLE_VERIFIER
2083                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2084                                 uc = iptr->sx.s23.s2.uc;
2085
2086                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2087                         }
2088 #endif /* ENABLE_VERIFIER */
2089
2090                         M_CALL_IMM(0);                            /* passing exception pc */
2091                         M_POP(REG_ITMP2_XPC);
2092
2093                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2094                         M_JMP(REG_ITMP3);
2095                         break;
2096
2097                 case ICMD_GOTO:         /* ... ==> ...                                */
2098                 case ICMD_RET:
2099
2100                         emit_br(cd, iptr->dst.block);
2101                         ALIGNCODENOP;
2102                         break;
2103
2104                 case ICMD_JSR:          /* ... ==> ...                                */
2105
2106                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2107                         ALIGNCODENOP;
2108                         break;
2109
2110                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2111                 case ICMD_IFNONNULL:
2112
2113                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2114                         M_TEST(s1);
2115                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2116                         break;
2117
2118                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2119                 case ICMD_IFLT:
2120                 case ICMD_IFLE:
2121                 case ICMD_IFNE:
2122                 case ICMD_IFGT:
2123                 case ICMD_IFGE:
2124
2125                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2126                         M_ICMP_IMM(iptr->sx.val.i, s1);
2127                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2128                         break;
2129
2130                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2131                 case ICMD_IF_LNE:
2132                 case ICMD_IF_LLT:
2133                 case ICMD_IF_LGE:
2134                 case ICMD_IF_LGT:
2135                 case ICMD_IF_LLE:
2136
2137                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2138                         if (IS_IMM32(iptr->sx.val.l))
2139                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2140                         else {
2141                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2142                                 M_LCMP(REG_ITMP2, s1);
2143                         }
2144                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2145                         break;
2146
2147                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2148                 case ICMD_IF_ICMPNE:
2149                 case ICMD_IF_ICMPLT:
2150                 case ICMD_IF_ICMPGE:
2151                 case ICMD_IF_ICMPGT:
2152                 case ICMD_IF_ICMPLE:
2153
2154                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2155                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2156                         M_ICMP(s2, s1);
2157                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2158                         break;
2159
2160                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2161                 case ICMD_IF_ACMPNE:
2162
2163                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2164                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2165                         M_LCMP(s2, s1);
2166                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2167                         break;
2168
2169                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2170                 case ICMD_IF_LCMPNE:
2171                 case ICMD_IF_LCMPLT:
2172                 case ICMD_IF_LCMPGE:
2173                 case ICMD_IF_LCMPGT:
2174                 case ICMD_IF_LCMPLE:
2175
2176                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2177                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2178                         M_LCMP(s2, s1);
2179                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2180                         break;
2181
2182                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2183                 case ICMD_LRETURN:
2184
2185                         REPLACEMENT_POINT_RETURN(cd, iptr);
2186                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2187                         M_INTMOVE(s1, REG_RESULT);
2188                         goto nowperformreturn;
2189
2190                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2191
2192                         REPLACEMENT_POINT_RETURN(cd, iptr);
2193                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2194                         M_INTMOVE(s1, REG_RESULT);
2195
2196 #ifdef ENABLE_VERIFIER
2197                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2198                                 uc = iptr->sx.s23.s2.uc;
2199
2200                                 PROFILE_CYCLE_STOP;
2201
2202                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2203
2204                                 PROFILE_CYCLE_START;
2205                         }
2206 #endif /* ENABLE_VERIFIER */
2207                         goto nowperformreturn;
2208
2209                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2210                 case ICMD_DRETURN:
2211
2212                         REPLACEMENT_POINT_RETURN(cd, iptr);
2213                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2214                         M_FLTMOVE(s1, REG_FRESULT);
2215                         goto nowperformreturn;
2216
2217                 case ICMD_RETURN:      /* ...  ==> ...                                */
2218
2219                         REPLACEMENT_POINT_RETURN(cd, iptr);
2220
2221 nowperformreturn:
2222                         {
2223                         s4 i, p;
2224
2225                         p = cd->stackframesize;
2226
2227 #if !defined(NDEBUG)
2228                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2229                                 emit_verbosecall_exit(jd);
2230 #endif /* !defined(NDEBUG) */
2231
2232 #if defined(ENABLE_THREADS)
2233                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2234                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2235
2236                                 /* we need to save the proper return value */
2237                                 switch (iptr->opc) {
2238                                 case ICMD_IRETURN:
2239                                 case ICMD_ARETURN:
2240                                 case ICMD_LRETURN:
2241                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2242                                         break;
2243                                 case ICMD_FRETURN:
2244                                 case ICMD_DRETURN:
2245                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2246                                         break;
2247                                 }
2248
2249                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2250                                 M_CALL(REG_ITMP1);
2251
2252                                 /* and now restore the proper return value */
2253                                 switch (iptr->opc) {
2254                                 case ICMD_IRETURN:
2255                                 case ICMD_ARETURN:
2256                                 case ICMD_LRETURN:
2257                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2258                                         break;
2259                                 case ICMD_FRETURN:
2260                                 case ICMD_DRETURN:
2261                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2262                                         break;
2263                                 }
2264                         }
2265 #endif
2266
2267                         /* restore saved registers */
2268
2269                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2270                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2271                         }
2272                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2273                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2274                         }
2275
2276                         /* deallocate stack */
2277
2278                         if (cd->stackframesize)
2279                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2280
2281                         /* generate method profiling code */
2282
2283                         PROFILE_CYCLE_STOP;
2284
2285                         M_RET;
2286                         }
2287                         break;
2288
2289
2290                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2291                         {
2292                                 s4 i, l;
2293                                 branch_target_t *table;
2294
2295                                 table = iptr->dst.table;
2296
2297                                 l = iptr->sx.s23.s2.tablelow;
2298                                 i = iptr->sx.s23.s3.tablehigh;
2299
2300                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2301                                 M_INTMOVE(s1, REG_ITMP1);
2302
2303                                 if (l != 0)
2304                                         M_ISUB_IMM(l, REG_ITMP1);
2305
2306                                 /* number of targets */
2307                                 i = i - l + 1;
2308
2309                 /* range check */
2310
2311                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2312                                 emit_bugt(cd, table[0].block);
2313
2314                                 /* build jump table top down and use address of lowest entry */
2315
2316                                 table += i;
2317
2318                                 while (--i >= 0) {
2319                                         dseg_add_target(cd, table->block);
2320                                         --table;
2321                                 }
2322
2323                                 /* length of dataseg after last dseg_add_target is used
2324                                    by load */
2325
2326                                 M_MOV_IMM(0, REG_ITMP2);
2327                                 dseg_adddata(cd);
2328                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2329                                 M_JMP(REG_ITMP1);
2330                         }
2331                         break;
2332
2333
2334                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2335                         {
2336                                 s4 i;
2337                                 lookup_target_t *lookup;
2338
2339                                 lookup = iptr->dst.lookup;
2340
2341                                 i = iptr->sx.s23.s2.lookupcount;
2342
2343                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2344                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2345
2346                                 while (--i >= 0) {
2347                                         M_ICMP_IMM(lookup->value, s1);
2348                                         emit_beq(cd, lookup->target.block);
2349                                         lookup++;
2350                                 }
2351
2352                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2353                                 ALIGNCODENOP;
2354                         }
2355                         break;
2356
2357
2358                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2359
2360                         bte = iptr->sx.s23.s3.bte;
2361                         md  = bte->md;
2362                         goto gen_method;
2363
2364                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2365
2366                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2367                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2368                 case ICMD_INVOKEINTERFACE:
2369
2370                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2371
2372                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2373                                 lm = NULL;
2374                                 um = iptr->sx.s23.s3.um;
2375                                 md = um->methodref->parseddesc.md;
2376                         }
2377                         else {
2378                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2379                                 um = NULL;
2380                                 md = lm->parseddesc;
2381                         }
2382
2383 gen_method:
2384                         s3 = md->paramcount;
2385
2386                         MCODECHECK((20 * s3) + 128);
2387
2388                         /* copy arguments to registers or stack location */
2389
2390                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2391                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2392
2393                                 /* Already Preallocated (ARGVAR) ? */
2394                                 if (var->flags & PREALLOC)
2395                                         continue;
2396
2397                                 if (IS_INT_LNG_TYPE(var->type)) {
2398                                         if (!md->params[s3].inmemory) {
2399                                                 s1 = rd->argintregs[md->params[s3].regoff];
2400                                                 d = emit_load(jd, iptr, var, s1);
2401                                                 M_INTMOVE(d, s1);
2402                                         }
2403                                         else {
2404                                                 d = emit_load(jd, iptr, var, REG_ITMP1);
2405                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
2406                                         }
2407                                 }
2408                                 else {
2409                                         if (!md->params[s3].inmemory) {
2410                                                 s1 = rd->argfltregs[md->params[s3].regoff];
2411                                                 d = emit_load(jd, iptr, var, s1);
2412                                                 M_FLTMOVE(d, s1);
2413                                         }
2414                                         else {
2415                                                 d = emit_load(jd, iptr, var, REG_FTMP1);
2416
2417                                                 if (IS_2_WORD_TYPE(var->type))
2418                                                         M_DST(d, REG_SP, md->params[s3].regoff * 8);
2419                                                 else
2420                                                         M_FST(d, REG_SP, md->params[s3].regoff * 8);
2421                                         }
2422                                 }
2423                         }
2424
2425                         /* generate method profiling code */
2426
2427                         PROFILE_CYCLE_STOP;
2428
2429                         switch (iptr->opc) {
2430                         case ICMD_BUILTIN:
2431                                 M_MOV_IMM(bte->fp, REG_ITMP1);
2432                                 M_CALL(REG_ITMP1);
2433
2434                                 emit_exception_check(cd, iptr);
2435                                 break;
2436
2437                         case ICMD_INVOKESPECIAL:
2438                                 emit_nullpointer_check(cd, iptr, REG_A0);
2439                                 /* fall through */
2440
2441                         case ICMD_INVOKESTATIC:
2442                                 if (lm == NULL) {
2443                                         disp = dseg_add_unique_address(cd, NULL);
2444                                         disp = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
2445
2446                                         /* must be calculated before codegen_add_patch_ref */
2447
2448                                         if (opt_shownops)
2449                                                 disp -= PATCHER_CALL_SIZE;
2450
2451                                         codegen_add_patch_ref(cd, PATCHER_invokestatic_special,
2452                                                                                   um, disp);
2453
2454 /*                                      a = 0; */
2455                                 }
2456                                 else {
2457                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2458                                         disp = disp + -((cd->mcodeptr + 7) - cd->mcodebase);
2459
2460 /*                                      a = (ptrint) lm->stubroutine; */
2461                                 }
2462
2463 /*                              M_MOV_IMM(a, REG_ITMP2); */
2464                                 M_ALD(REG_ITMP2, RIP, disp);
2465                                 M_CALL(REG_ITMP2);
2466                                 break;
2467
2468                         case ICMD_INVOKEVIRTUAL:
2469                                 if (lm == NULL) {
2470                                         codegen_add_patch_ref(cd, PATCHER_invokevirtual, um, 0);
2471
2472                                         s1 = 0;
2473                                 }
2474                                 else {
2475                                         s1 = OFFSET(vftbl_t, table[0]) +
2476                                                 sizeof(methodptr) * lm->vftblindex;
2477                                 }
2478
2479                                 /* implicit null-pointer check */
2480                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_objectheader, vftbl));
2481                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2482                                 M_CALL(REG_ITMP3);
2483                                 break;
2484
2485                         case ICMD_INVOKEINTERFACE:
2486                                 if (lm == NULL) {
2487                                         codegen_add_patch_ref(cd, PATCHER_invokeinterface, um, 0);
2488
2489                                         s1 = 0;
2490                                         s2 = 0;
2491                                 }
2492                                 else {
2493                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2494                                                 sizeof(methodptr) * lm->class->index;
2495
2496                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2497                                 }
2498
2499                                 /* implicit null-pointer check */
2500                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_objectheader, vftbl));
2501                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2502                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2503                                 M_CALL(REG_ITMP3);
2504                                 break;
2505                         }
2506
2507                         /* generate method profiling code */
2508
2509                         PROFILE_CYCLE_START;
2510
2511                         /* store size of call code in replacement point */
2512
2513                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2514
2515                         /* store return value */
2516
2517                         switch (md->returntype.type) {
2518                         case TYPE_INT:
2519                         case TYPE_LNG:
2520                         case TYPE_ADR:
2521                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2522                                 M_INTMOVE(REG_RESULT, s1);
2523                                 emit_store_dst(jd, iptr, s1);
2524                                 break;
2525                         case TYPE_FLT:
2526                         case TYPE_DBL:
2527                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2528                                 M_FLTMOVE(REG_FRESULT, s1);
2529                                 emit_store_dst(jd, iptr, s1);
2530                                 break;
2531                         default:
2532                                 /* TYPE_VOID */
2533                                 break;
2534                         }
2535                         break;
2536
2537
2538                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2539
2540                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2541                                 /* object type cast-check */
2542
2543                                 classinfo *super;
2544                                 vftbl_t   *supervftbl;
2545                                 s4         superindex;
2546
2547                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2548                                         super      = NULL;
2549                                         superindex = 0;
2550                                         supervftbl = NULL;
2551                                 }
2552                                 else {
2553                                         super      = iptr->sx.s23.s3.c.cls;
2554                                         superindex = super->index;
2555                                         supervftbl = super->vftbl;
2556                                 }
2557
2558 #if defined(ENABLE_THREADS)
2559                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
2560 #endif
2561                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2562
2563                                 /* if class is not resolved, check which code to call */
2564
2565                                 if (super == NULL) {
2566                                         M_TEST(s1);
2567                                         emit_label_beq(cd, BRANCH_LABEL_1);
2568
2569                                         codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2570                                                                                   iptr->sx.s23.s3.c.ref, 0);
2571
2572                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2573                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2574                                         emit_label_beq(cd, BRANCH_LABEL_2);
2575                                 }
2576
2577                                 /* interface checkcast code */
2578
2579                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2580                                         if (super != NULL) {
2581                                                 M_TEST(s1);
2582                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2583                                         }
2584
2585                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2586
2587                                         if (super == NULL) {
2588                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_interface,
2589                                                                                           iptr->sx.s23.s3.c.ref,
2590                                                                                           0);
2591                                         }
2592
2593                                         M_ILD32(REG_ITMP3,
2594                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2595                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2596                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2597
2598                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2599                                                         OFFSET(vftbl_t, interfacetable[0]) -
2600                                                         superindex * sizeof(methodptr*));
2601                                         M_TEST(REG_ITMP3);
2602                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2603
2604                                         if (super == NULL)
2605                                                 emit_label_br(cd, BRANCH_LABEL_4);
2606                                         else
2607                                                 emit_label(cd, BRANCH_LABEL_3);
2608                                 }
2609
2610                                 /* class checkcast code */
2611
2612                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2613                                         if (super == NULL) {
2614                                                 emit_label(cd, BRANCH_LABEL_2);
2615                                         }
2616                                         else {
2617                                                 M_TEST(s1);
2618                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2619                                         }
2620
2621                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2622
2623                                         if (super == NULL) {
2624                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_class,
2625                                                                                           iptr->sx.s23.s3.c.ref,
2626                                                                                           0);
2627                                         }
2628
2629                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2630 #if defined(ENABLE_THREADS)
2631                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
2632 #endif
2633                                         M_ILD32(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2634
2635                                         /*                                      if (s1 != REG_ITMP1) { */
2636                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2637                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2638                                         /*                                                                                              REG_ITMP1); */
2639                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2640                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2641                                         /*                                                                                              REG_ITMP3); */
2642                                         /*  #if defined(ENABLE_THREADS) */
2643                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2644                                         /*  #endif */
2645                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2646
2647                                         /*                                      } else { */
2648
2649                                         M_ILD32(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
2650                                         M_ISUB(REG_ITMP3, REG_ITMP2);
2651                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2652                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2653                                         /*                                      } */
2654 #if defined(ENABLE_THREADS)
2655                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
2656 #endif
2657                                         M_ICMP(REG_ITMP3, REG_ITMP2);
2658                                         emit_classcast_check(cd, iptr, BRANCH_UGT, REG_ITMP3, s1);
2659
2660                                         if (super != NULL)
2661                                                 emit_label(cd, BRANCH_LABEL_5);
2662                                 }
2663
2664                                 if (super == NULL) {
2665                                         emit_label(cd, BRANCH_LABEL_1);
2666                                         emit_label(cd, BRANCH_LABEL_4);
2667                                 }
2668
2669                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2670                         }
2671                         else {
2672                                 /* array type cast-check */
2673
2674                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2675                                 M_INTMOVE(s1, REG_A0);
2676
2677                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2678                                         codegen_add_patch_ref(cd, PATCHER_builtin_arraycheckcast,
2679                                                                                   iptr->sx.s23.s3.c.ref, 0);
2680                                 }
2681
2682                                 M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2683                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2684                                 M_CALL(REG_ITMP1);
2685
2686                                 /* s1 may have been destroyed over the function call */
2687                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2688                                 M_TEST(REG_RESULT);
2689                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2690
2691                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2692                         }
2693
2694                         M_INTMOVE(s1, d);
2695                         emit_store_dst(jd, iptr, d);
2696                         break;
2697
2698                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2699
2700                         {
2701                         classinfo *super;
2702                         vftbl_t   *supervftbl;
2703                         s4         superindex;
2704
2705                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2706                                 super      = NULL;
2707                                 superindex = 0;
2708                                 supervftbl = NULL;
2709
2710                         } else {
2711                                 super      = iptr->sx.s23.s3.c.cls;
2712                                 superindex = super->index;
2713                                 supervftbl = super->vftbl;
2714                         }
2715
2716 #if defined(ENABLE_THREADS)
2717             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
2718 #endif
2719
2720                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2721                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2722
2723                         if (s1 == d) {
2724                                 M_INTMOVE(s1, REG_ITMP1);
2725                                 s1 = REG_ITMP1;
2726                         }
2727
2728                         M_CLR(d);
2729
2730                         /* if class is not resolved, check which code to call */
2731
2732                         if (super == NULL) {
2733                                 M_TEST(s1);
2734                                 emit_label_beq(cd, BRANCH_LABEL_1);
2735
2736                                 codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2737                                                                           iptr->sx.s23.s3.c.ref, 0);
2738
2739                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2740                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2741                                 emit_label_beq(cd, BRANCH_LABEL_2);
2742                         }
2743
2744                         /* interface instanceof code */
2745
2746                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2747                                 if (super != NULL) {
2748                                         M_TEST(s1);
2749                                         emit_label_beq(cd, BRANCH_LABEL_3);
2750                                 }
2751
2752                                 M_ALD(REG_ITMP1, s1, OFFSET(java_objectheader, vftbl));
2753
2754                                 if (super == NULL) {
2755                                         codegen_add_patch_ref(cd, PATCHER_instanceof_interface,
2756                                                                                   iptr->sx.s23.s3.c.ref, 0);
2757                                 }
2758
2759                                 M_ILD32(REG_ITMP3,
2760                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2761                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2762
2763                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2764
2765                                 M_BLE(a);
2766                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2767                                                 OFFSET(vftbl_t, interfacetable[0]) -
2768                                                 superindex * sizeof(methodptr*));
2769                                 M_TEST(REG_ITMP1);
2770                                 M_SETNE(d);
2771
2772                                 if (super == NULL)
2773                                         emit_label_br(cd, BRANCH_LABEL_4);
2774                                 else
2775                                         emit_label(cd, BRANCH_LABEL_3);
2776                         }
2777
2778                         /* class instanceof code */
2779
2780                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2781                                 if (super == NULL) {
2782                                         emit_label(cd, BRANCH_LABEL_2);
2783                                 }
2784                                 else {
2785                                         M_TEST(s1);
2786                                         emit_label_beq(cd, BRANCH_LABEL_5);
2787                                 }
2788
2789                                 M_ALD(REG_ITMP1, s1, OFFSET(java_objectheader, vftbl));
2790
2791                                 if (super == NULL) {
2792                                         codegen_add_patch_ref(cd, PATCHER_instanceof_class,
2793                                                                                   iptr->sx.s23.s3.c.ref, 0);
2794                                 }
2795
2796                                 M_MOV_IMM(supervftbl, REG_ITMP2);
2797
2798 #if defined(ENABLE_THREADS)
2799                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
2800 #endif
2801
2802                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
2803                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2804                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2805
2806 #if defined(ENABLE_THREADS)
2807                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
2808 #endif
2809
2810                                 M_ISUB(REG_ITMP2, REG_ITMP1);
2811                                 M_CLR(d); /* may be REG_ITMP2 */
2812                                 M_ICMP(REG_ITMP3, REG_ITMP1);
2813                                 M_SETULE(d);
2814
2815                                 if (super != NULL)
2816                                         emit_label(cd, BRANCH_LABEL_5);
2817                         }
2818
2819                         if (super == NULL) {
2820                                 emit_label(cd, BRANCH_LABEL_1);
2821                                 emit_label(cd, BRANCH_LABEL_4);
2822                         }
2823
2824                         emit_store_dst(jd, iptr, d);
2825                         }
2826                         break;
2827
2828                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2829
2830                         /* check for negative sizes and copy sizes to stack if necessary  */
2831
2832                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2833
2834                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2835
2836                                 /* copy SAVEDVAR sizes to stack */
2837                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2838
2839                                 /* Already Preallocated? */
2840                                 if (!(var->flags & PREALLOC)) {
2841                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2842                                         M_LST(s2, REG_SP, s1 * 8);
2843                                 }
2844                         }
2845
2846                         /* is a patcher function set? */
2847
2848                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2849                                 codegen_add_patch_ref(cd, PATCHER_builtin_multianewarray,
2850                                                                           iptr->sx.s23.s3.c.ref, 0);
2851                         }
2852
2853                         /* a0 = dimension count */
2854
2855                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2856
2857                         /* a1 = classinfo */
2858
2859                         M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
2860
2861                         /* a2 = pointer to dimensions = stack pointer */
2862
2863                         M_MOV(REG_SP, REG_A2);
2864
2865                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2866                         M_CALL(REG_ITMP1);
2867
2868                         /* check for exception before result assignment */
2869
2870                         emit_exception_check(cd, iptr);
2871
2872                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2873                         M_INTMOVE(REG_RESULT, s1);
2874                         emit_store_dst(jd, iptr, s1);
2875                         break;
2876
2877                 default:
2878                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2879                                                                                    iptr->opc);
2880                         return false;
2881         } /* switch */
2882
2883         } /* for instruction */
2884
2885         MCODECHECK(512); /* XXX require a lower number? */
2886
2887         /* At the end of a basic block we may have to append some nops,
2888            because the patcher stub calling code might be longer than the
2889            actual instruction. So codepatching does not change the
2890            following block unintentionally. */
2891
2892         if (cd->mcodeptr < cd->lastmcodeptr) {
2893                 while (cd->mcodeptr < cd->lastmcodeptr) {
2894                         M_NOP;
2895                 }
2896         }
2897
2898         } /* if (bptr -> flags >= BBREACHED) */
2899         } /* for basic block */
2900
2901         dseg_createlinenumbertable(cd);
2902
2903         /* generate stubs */
2904
2905         emit_patcher_stubs(jd);
2906         REPLACEMENT_EMIT_STUBS(jd);
2907
2908         /* everything's ok */
2909
2910         return true;
2911 }
2912
2913
2914 /* codegen_emit_stub_compiler **************************************************
2915
2916    Emit a stub routine which calls the compiler.
2917
2918 *******************************************************************************/
2919
2920 void codegen_emit_stub_compiler(jitdata *jd)
2921 {
2922         methodinfo  *m;
2923         codegendata *cd;
2924
2925         /* get required compiler data */
2926
2927         m  = jd->m;
2928         cd = jd->cd;
2929
2930         /* code for the stub */
2931
2932         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
2933         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
2934         M_JMP(REG_ITMP3);
2935 }
2936
2937
2938 /* codegen_emit_stub_builtin ***************************************************
2939
2940    Creates a stub routine which calls a builtin function.
2941
2942 *******************************************************************************/
2943
2944 void codegen_emit_stub_builtin(jitdata *jd, functionptr f)
2945 {
2946         methodinfo  *m;
2947         codeinfo    *code;
2948         codegendata *cd;
2949         registerdata *rd; /* REMOVE ME */
2950         methoddesc  *md;
2951         s4           i, j;                 /* count variables                    */
2952         s4           s1, disp;
2953
2954         /* get required compiler data */
2955
2956         m    = jd->m;
2957         code = jd->code;
2958         cd   = jd->cd;
2959         rd   = jd->rd;
2960
2961         /* initialize variables */
2962
2963         md = m->parseddesc;
2964
2965         /* calculate stack frame size */
2966
2967         cd->stackframesize =
2968                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
2969                 1;                                                    /* return value */
2970
2971         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
2972
2973         /* create method header */
2974
2975         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2976         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2977         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
2978         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2979         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2980         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2981         (void) dseg_addlinenumbertablesize(cd);
2982         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
2983
2984         /* generate stub code */
2985
2986         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2987
2988 #if defined(ENABLE_GC_CACAO)
2989         /* Save callee saved integer registers in stackframeinfo (GC may
2990            need to recover them during a collection). */
2991
2992         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
2993                 OFFSET(stackframeinfo, intregs);
2994
2995         for (i = 0; i < INT_SAV_CNT; i++)
2996                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2997 #endif
2998
2999         /* save integer and float argument registers */
3000
3001         for (i = 0, j = 0; i < md->paramcount; i++) {
3002                 if (!md->params[i].inmemory) {
3003                         s1 = md->params[i].regoff;
3004
3005                         switch (md->paramtypes[i].type) {
3006                         case TYPE_INT:
3007                         case TYPE_LNG:
3008                         case TYPE_ADR:
3009                                 M_LST(rd->argintregs[s1], REG_SP, j * 8);
3010                                 break;
3011                         case TYPE_FLT:
3012                         case TYPE_DBL:
3013                                 M_DST(rd->argfltregs[s1], REG_SP, j * 8);
3014                                 break;
3015                         }
3016
3017                         j++;
3018                 }
3019         }
3020
3021         /* create dynamic stack info */
3022
3023         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3024         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3025         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3026         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3027         M_MOV_IMM(codegen_stub_builtin_enter, REG_ITMP1);
3028         M_CALL(REG_ITMP1);
3029
3030         /* restore integer and float argument registers */
3031
3032         for (i = 0, j = 0; i < md->paramcount; i++) {
3033                 if (!md->params[i].inmemory) {
3034                         s1 = md->params[i].regoff;
3035
3036                         switch (md->paramtypes[i].type) {
3037                         case TYPE_INT:
3038                         case TYPE_LNG:
3039                         case TYPE_ADR:
3040                                 M_LLD(abi_registers_integer_argument[s1], REG_SP, j * 8);
3041                                 break;
3042                         case TYPE_FLT:
3043                         case TYPE_DBL:
3044                                 M_DLD(abi_registers_integer_argument[s1], REG_SP, j * 8);
3045                                 break;
3046                         }
3047
3048                         j++;
3049                 }
3050         }
3051
3052         /* call the builtin function */
3053
3054         M_MOV_IMM(f, REG_ITMP3);
3055         M_CALL(REG_ITMP3);
3056
3057         /* save return value */
3058
3059         switch (md->returntype.type) {
3060         case TYPE_INT:
3061         case TYPE_LNG:
3062         case TYPE_ADR:
3063                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3064                 break;
3065         case TYPE_FLT:
3066         case TYPE_DBL:
3067                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3068                 break;
3069         case TYPE_VOID:
3070                 break;
3071         }
3072
3073         /* remove native stackframe info */
3074
3075         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3076         M_MOV_IMM(codegen_stub_builtin_exit, REG_ITMP1);
3077         M_CALL(REG_ITMP1);
3078
3079         /* restore return value */
3080
3081         switch (md->returntype.type) {
3082         case TYPE_INT:
3083         case TYPE_LNG:
3084         case TYPE_ADR:
3085                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3086                 break;
3087         case TYPE_FLT:
3088         case TYPE_DBL:
3089                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3090                 break;
3091         case TYPE_VOID:
3092                 break;
3093         }
3094
3095 #if defined(ENABLE_GC_CACAO)
3096         /* Restore callee saved integer registers from stackframeinfo (GC
3097            might have modified them during a collection). */
3098
3099         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3100                 OFFSET(stackframeinfo, intregs);
3101
3102         for (i = 0; i < INT_SAV_CNT; i++)
3103                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3104 #endif
3105
3106         /* remove stackframe */
3107
3108         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3109         M_RET;
3110 }
3111
3112
3113 /* codegen_emit_stub_native ****************************************************
3114
3115    Emits a stub routine which calls a native method.
3116
3117 *******************************************************************************/
3118
3119 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f)
3120 {
3121         methodinfo   *m;
3122         codeinfo     *code;
3123         codegendata  *cd;
3124         registerdata *rd;
3125         methoddesc   *md;
3126         s4            nativeparams;
3127         s4            i, j;                 /* count variables                    */
3128         s4            t;
3129         s4            s1, s2, disp;
3130
3131         /* get required compiler data */
3132
3133         m    = jd->m;
3134         code = jd->code;
3135         cd   = jd->cd;
3136         rd   = jd->rd;
3137
3138         /* initialize variables */
3139
3140         md = m->parseddesc;
3141         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
3142
3143         /* calculate stack frame size */
3144
3145         cd->stackframesize =
3146                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
3147                 sizeof(localref_table) / SIZEOF_VOID_P +
3148                 INT_ARG_CNT + FLT_ARG_CNT +
3149                 1 +                       /* functionptr, TODO: store in data segment */
3150                 nmd->memuse;
3151
3152         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
3153
3154         /* create method header */
3155
3156         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
3157         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
3158         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
3159         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
3160         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
3161         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
3162         (void) dseg_addlinenumbertablesize(cd);
3163         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
3164
3165 #if defined(ENABLE_PROFILING)
3166         /* generate native method profiling code */
3167
3168         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3169                 /* count frequency */
3170
3171                 M_MOV_IMM(code, REG_ITMP3);
3172                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3173         }
3174 #endif
3175
3176         /* generate stub code */
3177
3178         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3179
3180 #if !defined(NDEBUG)
3181         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3182                 emit_verbosecall_enter(jd);
3183 #endif
3184
3185         /* get function address (this must happen before the stackframeinfo) */
3186
3187 #if !defined(WITH_STATIC_CLASSPATH)
3188         if (f == NULL)
3189                 codegen_add_patch_ref(cd, PATCHER_resolve_native, m, 0);
3190 #endif
3191
3192         M_MOV_IMM(f, REG_ITMP3);
3193
3194 #if defined(ENABLE_GC_CACAO)
3195         /* Save callee saved integer registers in stackframeinfo (GC may
3196            need to recover them during a collection). */
3197
3198         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3199                 OFFSET(stackframeinfo, intregs);
3200
3201         for (i = 0; i < INT_SAV_CNT; i++)
3202                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3203 #endif
3204
3205         /* save integer and float argument registers */
3206
3207         for (i = 0, j = 0; i < md->paramcount; i++) {
3208                 if (!md->params[i].inmemory) {
3209                         s1 = md->params[i].regoff;
3210
3211                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3212                                 M_LST(rd->argintregs[s1], REG_SP, j * 8);
3213                         else
3214                                 M_DST(rd->argfltregs[s1], REG_SP, j * 8);
3215
3216                         j++;
3217                 }
3218         }
3219
3220         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3221
3222         /* create dynamic stack info */
3223
3224         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3225         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3226         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3227         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3228         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3229         M_CALL(REG_ITMP1);
3230
3231         /* restore integer and float argument registers */
3232
3233         for (i = 0, j = 0; i < md->paramcount; i++) {
3234                 if (!md->params[i].inmemory) {
3235                         s1 = md->params[i].regoff;
3236
3237                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3238                                 M_LLD(rd->argintregs[s1], REG_SP, j * 8);
3239                         else
3240                                 M_DLD(rd->argfltregs[s1], REG_SP, j * 8);
3241
3242                         j++;
3243                 }
3244         }
3245
3246         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3247
3248
3249         /* copy or spill arguments to new locations */
3250
3251         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3252                 t = md->paramtypes[i].type;
3253
3254                 if (IS_INT_LNG_TYPE(t)) {
3255                         if (!md->params[i].inmemory) {
3256                                 s1 = rd->argintregs[md->params[i].regoff];
3257
3258                                 if (!nmd->params[j].inmemory) {
3259                                         s2 = rd->argintregs[nmd->params[j].regoff];
3260                                         M_INTMOVE(s1, s2);
3261
3262                                 } else {
3263                                         s2 = nmd->params[j].regoff;
3264                                         M_LST(s1, REG_SP, s2 * 8);
3265                                 }
3266
3267                         } else {
3268                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3269                                 s2 = nmd->params[j].regoff;
3270                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
3271                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
3272                         }
3273
3274                 } else {
3275                         /* We only copy spilled float arguments, as the float argument    */
3276                         /* registers keep unchanged.                                      */
3277
3278                         if (md->params[i].inmemory) {
3279                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3280                                 s2 = nmd->params[j].regoff;
3281
3282                                 if (IS_2_WORD_TYPE(t)) {
3283                                         M_DLD(REG_FTMP1, REG_SP, s1 * 8);
3284                                         M_DST(REG_FTMP1, REG_SP, s2 * 8);
3285                                 } else {
3286                                         M_FLD(REG_FTMP1, REG_SP, s1 * 8);
3287                                         M_FST(REG_FTMP1, REG_SP, s2 * 8);
3288                                 }
3289                         }
3290                 }
3291         }
3292
3293         /* put class into second argument register */
3294
3295         if (m->flags & ACC_STATIC)
3296                 M_MOV_IMM(m->class, REG_A1);
3297
3298         /* put env into first argument register */
3299
3300         M_MOV_IMM(_Jv_env, REG_A0);
3301
3302         /* do the native function call */
3303
3304         M_CALL(REG_ITMP3);
3305
3306         /* save return value */
3307
3308         if (md->returntype.type != TYPE_VOID) {
3309                 if (IS_INT_LNG_TYPE(md->returntype.type))
3310                         M_LST(REG_RESULT, REG_SP, 0 * 8);
3311                 else
3312                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
3313         }
3314
3315 #if !defined(NDEBUG)
3316         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3317                 emit_verbosecall_exit(jd);
3318 #endif
3319
3320         /* remove native stackframe info */
3321
3322         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3323         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3324         M_CALL(REG_ITMP1);
3325         M_MOV(REG_RESULT, REG_ITMP3);
3326
3327         /* restore return value */
3328
3329         if (md->returntype.type != TYPE_VOID) {
3330                 if (IS_INT_LNG_TYPE(md->returntype.type))
3331                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
3332                 else
3333                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3334         }
3335
3336 #if defined(ENABLE_GC_CACAO)
3337         /* Restore callee saved integer registers from stackframeinfo (GC
3338            might have modified them during a collection). */
3339
3340         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3341                 OFFSET(stackframeinfo, intregs);
3342
3343         for (i = 0; i < INT_SAV_CNT; i++)
3344                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3345 #endif
3346
3347         /* remove stackframe */
3348
3349         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3350
3351         /* test for exception */
3352
3353         M_TEST(REG_ITMP3);
3354         M_BNE(1);
3355         M_RET;
3356
3357         /* handle exception */
3358
3359         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3360         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3361         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3362
3363         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3364         M_JMP(REG_ITMP3);
3365
3366         /* generate patcher stubs */
3367
3368         emit_patcher_stubs(jd);
3369 }
3370
3371
3372 /*
3373  * These are local overrides for various environment variables in Emacs.
3374  * Please do not remove this and leave it at the end of the file, where
3375  * Emacs will automagically detect them.
3376  * ---------------------------------------------------------------------
3377  * Local variables:
3378  * mode: c
3379  * indent-tabs-mode: t
3380  * c-basic-offset: 4
3381  * tab-width: 4
3382  * End:
3383  * vim:noexpandtab:sw=4:ts=4:
3384  */