src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25 */
  26
  27
  28 #include "config.h"
  29
  30 #include <assert.h>
  31 #include <stdio.h>
  32
  33 #include "vm/types.h"
  34
  35 #include "md-abi.h"
  36
  37 #include "vm/jit/x86_64/arch.h"
  38 #include "vm/jit/x86_64/codegen.h"
  39 #include "vm/jit/x86_64/emit.h"
  40
  41 #include "mm/memory.h"
  42
  43 #include "native/jni.h"
  44 #include "native/localref.h"
  45 #include "native/native.h"
  46
  47 #include "threads/lock-common.h"
  48
  49 #include "vm/builtin.h"
  50 #include "vm/exceptions.h"
  51 #include "vm/global.h"
  52 #include "vm/stringlocal.h"
  53 #include "vm/vm.h"
  54
  55 #include "vm/jit/abi.h"
  56 #include "vm/jit/asmpart.h"
  57 #include "vm/jit/codegen-common.h"
  58 #include "vm/jit/dseg.h"
  59 #include "vm/jit/emit-common.h"
  60 #include "vm/jit/jit.h"
  61 #include "vm/jit/methodheader.h"
  62 #include "vm/jit/parse.h"
  63 #include "vm/jit/patcher-common.h"
  64 #include "vm/jit/reg.h"
  65 #include "vm/jit/replace.h"
  66 #include "vm/jit/stacktrace.h"
  67
  68 #if defined(ENABLE_LSRA)
  69 # include "vm/jit/allocator/lsra.h"
  70 #endif
  71
  72 #include "vmcore/loader.h"
  73 #include "vmcore/options.h"
  74 #include "vmcore/statistics.h"
  75
  76
  77 /* codegen_emit ****************************************************************
  78
  79    Generates machine code.
  80
  81 *******************************************************************************/
  82
  83 bool codegen_emit(jitdata *jd)
  84 {
  85         methodinfo         *m;
  86         codeinfo           *code;
  87         codegendata        *cd;
  88         registerdata       *rd;
  89         s4                  len, s1, s2, s3, d, disp;
  90         u2                  currentline;
  91         ptrint              a;
  92         varinfo            *var, *dst;
  93         basicblock         *bptr;
  94         instruction        *iptr;
  95         exception_entry    *ex;
  96         constant_classref  *cr;
  97         unresolved_class   *uc;
  98         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
  99         unresolved_method  *um;
 100         builtintable_entry *bte;
 101         methoddesc         *md;
 102         fieldinfo          *fi;
 103         unresolved_field   *uf;
 104         s4                  fieldtype;
 105         s4                 varindex;
 106
 107         /* get required compiler data */
 108
 109         m    = jd->m;
 110         code = jd->code;
 111         cd   = jd->cd;
 112         rd   = jd->rd;
 113
 114         /* prevent compiler warnings */
 115
 116         d   = 0;
 117         lm  = NULL;
 118         um  = NULL;
 119         bte = NULL;
 120
 121         {
 122         s4 i, p, t, l;
 123         s4 savedregs_num;
 124
 125         savedregs_num = 0;
 126
 127         /* space to save used callee saved registers */
 128
 129         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 130         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 131
 132         cd->stackframesize = rd->memuse + savedregs_num;
 133
 134 #if defined(ENABLE_THREADS)
 135         /* space to save argument of monitor_enter */
 136
 137         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 138                 cd->stackframesize++;
 139 #endif
 140
 141         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 142            native code e.g. libc or jni (alignment problems with
 143            movaps). */
 144
 145         if (!jd->isleafmethod || opt_verbosecall)
 146                 cd->stackframesize |= 0x1;
 147
 148         /* create method header */
 149
 150         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 151         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 152
 153 #if defined(ENABLE_THREADS)
 154         /* IsSync contains the offset relative to the stack pointer for the
 155            argument of monitor_exit used in the exception handler. Since the
 156            offset could be zero and give a wrong meaning of the flag it is
 157            offset by one.
 158         */
 159
 160         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 161                 (void) dseg_add_unique_s4(cd, (rd->memuse + 1) * 8); /* IsSync        */
 162         else
 163 #endif
 164                 (void) dseg_add_unique_s4(cd, 0);                  /* IsSync          */
 165
 166         (void) dseg_add_unique_s4(cd, jd->isleafmethod);               /* IsLeaf  */
 167         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 168         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 169
 170         (void) dseg_addlinenumbertablesize(cd);
 171
 172         (void) dseg_add_unique_s4(cd, jd->exceptiontablelength); /* ExTableSize   */
 173
 174         /* create exception table */
 175
 176         for (ex = jd->exceptiontable; ex != NULL; ex = ex->down) {
 177                 dseg_add_target(cd, ex->start);
 178                 dseg_add_target(cd, ex->end);
 179                 dseg_add_target(cd, ex->handler);
 180                 (void) dseg_add_unique_address(cd, ex->catchtype.any);
 181         }
 182
 183 #if defined(ENABLE_PROFILING)
 184         /* generate method profiling code */
 185
 186         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 187                 /* count frequency */
 188
 189                 M_MOV_IMM(code, REG_ITMP3);
 190                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 191
 192                 PROFILE_CYCLE_START;
 193         }
 194 #endif
 195
 196         /* create stack frame (if necessary) */
 197
 198         if (cd->stackframesize)
 199                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 200
 201         /* save used callee saved registers */
 202
 203         p = cd->stackframesize;
 204         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 205                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 206         }
 207         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 208                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 209         }
 210
 211         /* take arguments out of register or stack frame */
 212
 213         md = m->parseddesc;
 214
 215         for (p = 0, l = 0; p < md->paramcount; p++) {
 216                 t = md->paramtypes[p].type;
 217
 218                 varindex = jd->local_map[l * 5 + t];
 219
 220                 l++;
 221                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 222                         l++;
 223
 224                 if (varindex == UNUSED)
 225                         continue;
 226
 227                 var = VAR(varindex);
 228
 229                 s1 = md->params[p].regoff;
 230
 231                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 232                         if (!md->params[p].inmemory) {           /* register arguments    */
 233                                 if (!IS_INMEMORY(var->flags))
 234                                         M_INTMOVE(s1, var->vv.regoff);
 235                                 else
 236                                     M_LST(s1, REG_SP, var->vv.regoff);
 237                         }
 238                         else {                                 /* stack arguments       */
 239                                 if (!IS_INMEMORY(var->flags))
 240                                         /* + 8 for return address */
 241                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 242                                 else
 243                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 244                         }
 245                 }
 246                 else {                                     /* floating args         */
 247                         if (!md->params[p].inmemory) {           /* register arguments    */
 248                                 if (!IS_INMEMORY(var->flags))
 249                                         M_FLTMOVE(s1, var->vv.regoff);
 250                                 else
 251                                         M_DST(s1, REG_SP, var->vv.regoff);
 252                         }
 253                         else {                                 /* stack arguments       */
 254                                 if (!IS_INMEMORY(var->flags))
 255                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 256                                 else
 257                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 258                         }
 259                 }
 260         }
 261
 262         /* save monitorenter argument */
 263
 264 #if defined(ENABLE_THREADS)
 265         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 266                 /* stack offset for monitor argument */
 267
 268                 s1 = rd->memuse;
 269
 270                 if (opt_verbosecall) {
 271                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 272
 273                         for (p = 0; p < INT_ARG_CNT; p++)
 274                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 275
 276                         for (p = 0; p < FLT_ARG_CNT; p++)
 277                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 278
 279                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 280                 }
 281
 282                 /* decide which monitor enter function to call */
 283
 284                 if (m->flags & ACC_STATIC) {
 285                         M_MOV_IMM(&m->class->object.header, REG_A0);
 286                 }
 287                 else {
 288                         M_TEST(REG_A0);
 289                         M_BNE(8);
 290                         M_ALD_MEM(REG_A0, EXCEPTION_HARDWARE_NULLPOINTER);
 291                 }
 292
 293                 M_AST(REG_A0, REG_SP, s1 * 8);
 294                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 295                 M_CALL(REG_ITMP1);
 296
 297                 if (opt_verbosecall) {
 298                         for (p = 0; p < INT_ARG_CNT; p++)
 299                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 300
 301                         for (p = 0; p < FLT_ARG_CNT; p++)
 302                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 303
 304                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 305                 }
 306         }
 307 #endif
 308
 309 #if !defined(NDEBUG)
 310         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 311                 emit_verbosecall_enter(jd);
 312 #endif /* !defined(NDEBUG) */
 313
 314         }
 315
 316         /* end of header generation */
 317
 318         /* create replacement points */
 319
 320         REPLACEMENT_POINTS_INIT(cd, jd);
 321
 322         /* walk through all basic blocks */
 323
 324         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 325
 326                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 327
 328                 if (bptr->flags >= BBREACHED) {
 329
 330                 /* branch resolving */
 331
 332                 codegen_resolve_branchrefs(cd, bptr);
 333
 334                 /* handle replacement points */
 335
 336                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 337
 338                 /* copy interface registers to their destination */
 339
 340                 len = bptr->indepth;
 341                 MCODECHECK(512);
 342
 343 #if defined(ENABLE_PROFILING)
 344                 /* generate basicblock profiling code */
 345
 346                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 347                         /* count frequency */
 348
 349                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 350                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 351
 352                         /* if this is an exception handler, start profiling again */
 353
 354                         if (bptr->type == BBTYPE_EXH)
 355                                 PROFILE_CYCLE_START;
 356                 }
 357 #endif
 358
 359 #if defined(ENABLE_LSRA)
 360                 if (opt_lsra) {
 361                         while (len) {
 362                                 len--;
 363                                 src = bptr->invars[len];
 364                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 365                                         if (bptr->type == BBTYPE_EXH) {
 366 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 367                                                 if (!IS_INMEMORY(src->flags))
 368                                                         d= src->vv.regoff;
 369                                                 else
 370                                                         d=REG_ITMP1;
 371                                                 M_INTMOVE(REG_ITMP1, d);
 372                                                 emit_store(jd, NULL, src, d);
 373                                         }
 374                                 }
 375                         }
 376
 377                 } else {
 378 #endif
 379
 380                 while (len) {
 381                         len--;
 382                         var = VAR(bptr->invars[len]);
 383                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 384                                 if (bptr->type == BBTYPE_EXH) {
 385                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 386                                         M_INTMOVE(REG_ITMP1, d);
 387                                         emit_store(jd, NULL, var, d);
 388                                 }
 389                         }
 390                         else {
 391                                 assert((var->flags & INOUT));
 392                         }
 393                 }
 394 #if defined(ENABLE_LSRA)
 395                 }
 396 #endif
 397                 /* walk through all instructions */
 398
 399                 len = bptr->icount;
 400                 currentline = 0;
 401
 402                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 403                         if (iptr->line != currentline) {
 404                                 dseg_addlinenumber(cd, iptr->line);
 405                                 currentline = iptr->line;
 406                         }
 407
 408                         MCODECHECK(1024);                         /* 1KB should be enough */
 409
 410                 switch (iptr->opc) {
 411                 case ICMD_NOP:        /* ...  ==> ...                                 */
 412                 case ICMD_POP:        /* ..., value  ==> ...                          */
 413                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 414                         break;
 415
 416                 case ICMD_INLINE_START:
 417
 418                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 419                         break;
 420
 421                 case ICMD_INLINE_BODY:
 422
 423                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 424                         dseg_addlinenumber_inline_start(cd, iptr);
 425                         dseg_addlinenumber(cd, iptr->line);
 426                         break;
 427
 428                 case ICMD_INLINE_END:
 429
 430                         dseg_addlinenumber_inline_end(cd, iptr);
 431                         dseg_addlinenumber(cd, iptr->line);
 432                         break;
 433
 434                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 435
 436                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 437                         emit_nullpointer_check(cd, iptr, s1);
 438                         break;
 439
 440                 /* constant operations ************************************************/
 441
 442                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 443
 444                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 445                         ICONST(d, iptr->sx.val.i);
 446                         emit_store_dst(jd, iptr, d);
 447                         break;
 448
 449                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 450
 451                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 452                         LCONST(d, iptr->sx.val.l);
 453                         emit_store_dst(jd, iptr, d);
 454                         break;
 455
 456                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 457
 458                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 459                         disp = dseg_add_float(cd, iptr->sx.val.f);
 460                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 461                         emit_store_dst(jd, iptr, d);
 462                         break;
 463
 464                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 465
 466                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 467                         disp = dseg_add_double(cd, iptr->sx.val.d);
 468                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 469                         emit_store_dst(jd, iptr, d);
 470                         break;
 471
 472                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 473
 474                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 475
 476                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 477                                 cr   = iptr->sx.val.c.ref;
 478                                 disp = dseg_add_unique_address(cd, cr);
 479
 480 /*                              PROFILE_CYCLE_STOP; */
 481
 482                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
 483                                                                           cr, disp);
 484
 485 /*                              PROFILE_CYCLE_START; */
 486
 487                                 M_ALD(d, RIP, disp);
 488                         }
 489                         else {
 490                                 if (iptr->sx.val.anyptr == 0) {
 491                                         M_CLR(d);
 492                                 }
 493                                 else {
 494                                         disp = dseg_add_address(cd, iptr->sx.val.anyptr);
 495                                         M_ALD(d, RIP, disp);
 496                                 }
 497                         }
 498                         emit_store_dst(jd, iptr, d);
 499                         break;
 500
 501
 502                 /* load/store/copy/move operations ************************************/
 503
 504                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 505                 case ICMD_ALOAD:      /* s1 = local variable                          */
 506                 case ICMD_LLOAD:
 507                 case ICMD_FLOAD:
 508                 case ICMD_DLOAD:
 509                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 510                 case ICMD_LSTORE:
 511                 case ICMD_FSTORE:
 512                 case ICMD_DSTORE:
 513                 case ICMD_COPY:
 514                 case ICMD_MOVE:
 515
 516                         emit_copy(jd, iptr);
 517                         break;
 518
 519                 case ICMD_ASTORE:
 520                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 521                                 emit_copy(jd, iptr);
 522                         break;
 523
 524                 /* integer operations *************************************************/
 525
 526                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 527
 528                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 529                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 530                         M_INTMOVE(s1, d);
 531                         M_INEG(d);
 532                         emit_store_dst(jd, iptr, d);
 533                         break;
 534
 535                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 536
 537                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 538                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 539                         M_INTMOVE(s1, d);
 540                         M_LNEG(d);
 541                         emit_store_dst(jd, iptr, d);
 542                         break;
 543
 544                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 545
 546                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 547                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 548                         M_ISEXT(s1, d);
 549                         emit_store_dst(jd, iptr, d);
 550                         break;
 551
 552                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 553
 554                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 555                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 556                         M_IMOV(s1, d);
 557                         emit_store_dst(jd, iptr, d);
 558                         break;
 559
 560                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 561
 562                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 563                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 564                         M_BSEXT(s1, d);
 565                         emit_store_dst(jd, iptr, d);
 566                         break;
 567
 568                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 569
 570                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 571                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 572                         M_CZEXT(s1, d);
 573                         emit_store_dst(jd, iptr, d);
 574                         break;
 575
 576                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 577
 578                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 579                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 580                         M_SSEXT(s1, d);
 581                         emit_store_dst(jd, iptr, d);
 582                         break;
 583
 584
 585                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 586
 587                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 588                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 589                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 590                         if (s2 == d)
 591                                 M_IADD(s1, d);
 592                         else {
 593                                 M_INTMOVE(s1, d);
 594                                 M_IADD(s2, d);
 595                         }
 596                         emit_store_dst(jd, iptr, d);
 597                         break;
 598
 599                 case ICMD_IINC:
 600                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 601                                       /* sx.val.i = constant                             */
 602
 603                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 604                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 605
 606                         /* Using inc and dec is not faster than add (tested with
 607                            sieve). */
 608
 609                         M_INTMOVE(s1, d);
 610                         M_IADD_IMM(iptr->sx.val.i, d);
 611                         emit_store_dst(jd, iptr, d);
 612                         break;
 613
 614                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 615
 616                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 617                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 618                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 619                         if (s2 == d)
 620                                 M_LADD(s1, d);
 621                         else {
 622                                 M_INTMOVE(s1, d);
 623                                 M_LADD(s2, d);
 624                         }
 625                         emit_store_dst(jd, iptr, d);
 626                         break;
 627
 628                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 629                                       /* sx.val.l = constant                             */
 630
 631                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 632                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 633                         M_INTMOVE(s1, d);
 634                         if (IS_IMM32(iptr->sx.val.l))
 635                                 M_LADD_IMM(iptr->sx.val.l, d);
 636                         else {
 637                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 638                                 M_LADD(REG_ITMP2, d);
 639                         }
 640                         emit_store_dst(jd, iptr, d);
 641                         break;
 642
 643                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 644
 645                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 646                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 647                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 648                         if (s2 == d) {
 649                                 M_INTMOVE(s1, REG_ITMP1);
 650                                 M_ISUB(s2, REG_ITMP1);
 651                                 M_INTMOVE(REG_ITMP1, d);
 652                         } else {
 653                                 M_INTMOVE(s1, d);
 654                                 M_ISUB(s2, d);
 655                         }
 656                         emit_store_dst(jd, iptr, d);
 657                         break;
 658
 659                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 660                                       /* sx.val.i = constant                             */
 661
 662                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 663                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 664                         M_INTMOVE(s1, d);
 665                         M_ISUB_IMM(iptr->sx.val.i, d);
 666                         emit_store_dst(jd, iptr, d);
 667                         break;
 668
 669                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 670
 671                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 672                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 673                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 674                         if (s2 == d) {
 675                                 M_INTMOVE(s1, REG_ITMP1);
 676                                 M_LSUB(s2, REG_ITMP1);
 677                                 M_INTMOVE(REG_ITMP1, d);
 678                         } else {
 679                                 M_INTMOVE(s1, d);
 680                                 M_LSUB(s2, d);
 681                         }
 682                         emit_store_dst(jd, iptr, d);
 683                         break;
 684
 685                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 686                                       /* sx.val.l = constant                             */
 687
 688                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 689                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 690                         M_INTMOVE(s1, d);
 691                         if (IS_IMM32(iptr->sx.val.l))
 692                                 M_LSUB_IMM(iptr->sx.val.l, d);
 693                         else {
 694                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 695                                 M_LSUB(REG_ITMP2, d);
 696                         }
 697                         emit_store_dst(jd, iptr, d);
 698                         break;
 699
 700                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 701
 702                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 703                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 704                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 705                         if (s2 == d)
 706                                 M_IMUL(s1, d);
 707                         else {
 708                                 M_INTMOVE(s1, d);
 709                                 M_IMUL(s2, d);
 710                         }
 711                         emit_store_dst(jd, iptr, d);
 712                         break;
 713
 714                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 715                                       /* sx.val.i = constant                             */
 716
 717                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 718                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 719                         if (iptr->sx.val.i == 2) {
 720                                 M_INTMOVE(s1, d);
 721                                 M_ISLL_IMM(1, d);
 722                         } else
 723                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 724                         emit_store_dst(jd, iptr, d);
 725                         break;
 726
 727                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 728
 729                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 730                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 731                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 732                         if (s2 == d)
 733                                 M_LMUL(s1, d);
 734                         else {
 735                                 M_INTMOVE(s1, d);
 736                                 M_LMUL(s2, d);
 737                         }
 738                         emit_store_dst(jd, iptr, d);
 739                         break;
 740
 741                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 742                                       /* sx.val.l = constant                             */
 743
 744                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 745                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 746                         if (IS_IMM32(iptr->sx.val.l))
 747                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 748                         else {
 749                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 750                                 M_INTMOVE(s1, d);
 751                                 M_LMUL(REG_ITMP2, d);
 752                         }
 753                         emit_store_dst(jd, iptr, d);
 754                         break;
 755
 756                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 757
 758                         s1 = emit_load_s1(jd, iptr, RAX);
 759                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 760                         d = codegen_reg_of_dst(jd, iptr, RAX);
 761
 762                         M_INTMOVE(s1, RAX);
 763                         M_INTMOVE(s2, REG_ITMP3);
 764                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 765
 766                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 767
 768                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 769                         M_BNE(4 + 6);
 770                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 771                         M_BEQ(1 + 3);                                          /* 6 bytes */
 772
 773                         emit_cltd(cd);                                         /* 1 byte  */
 774                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 775
 776                         M_INTMOVE(RAX, d);
 777                         emit_store_dst(jd, iptr, d);
 778                         dst = VAROP(iptr->dst);
 779                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 780                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 781                         break;
 782
 783                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 784
 785                         s1 = emit_load_s1(jd, iptr, RAX);
 786                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 787                         d = codegen_reg_of_dst(jd, iptr, RDX);
 788
 789                         M_INTMOVE(s1, RAX);
 790                         M_INTMOVE(s2, REG_ITMP3);
 791                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 792
 793                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 794
 795                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 796                         M_BNE(3 + 4 + 6);
 797                         M_CLR(RDX);                                            /* 3 bytes */
 798                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 799                         M_BEQ(1 + 3);                                          /* 6 bytes */
 800
 801                         emit_cltd(cd);                                         /* 1 byte  */
 802                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 803
 804                         M_INTMOVE(RDX, d);
 805                         emit_store_dst(jd, iptr, d);
 806                         dst = VAROP(iptr->dst);
 807                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 808                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 809                         break;
 810
 811                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 812                                       /* sx.val.i = constant                             */
 813
 814                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 815                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 816                         M_INTMOVE(s1, REG_ITMP1);
 817                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 818                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 819                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 820                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 821                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 822                         emit_store_dst(jd, iptr, d);
 823                         break;
 824
 825                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 826                                       /* sx.val.i = constant                             */
 827
 828                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 829                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 830                         M_INTMOVE(s1, REG_ITMP1);
 831                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 832                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 833                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 834                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 835                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 836                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 837                         emit_store_dst(jd, iptr, d);
 838                         break;
 839
 840
 841                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 842
 843                         s1 = emit_load_s1(jd, iptr, RAX);
 844                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 845                         d = codegen_reg_of_dst(jd, iptr, RAX);
 846
 847                         M_INTMOVE(s1, RAX);
 848                         M_INTMOVE(s2, REG_ITMP3);
 849                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 850
 851                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 852
 853                         /* check as described in jvm spec */
 854                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 855                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 856                         M_BNE(4 + 6);
 857                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 858                         M_BEQ(2 + 3);                                          /* 6 bytes */
 859
 860                         emit_cqto(cd);                                         /* 2 bytes */
 861                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 862
 863                         M_INTMOVE(RAX, d);
 864                         emit_store_dst(jd, iptr, d);
 865                         dst = VAROP(iptr->dst);
 866                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 867                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 868                         break;
 869
 870                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 871
 872                         s1 = emit_load_s1(jd, iptr, RAX);
 873                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 874                         d = codegen_reg_of_dst(jd, iptr, RDX);
 875
 876                         M_INTMOVE(s1, RAX);
 877                         M_INTMOVE(s2, REG_ITMP3);
 878                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 879
 880                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 881
 882                         /* check as described in jvm spec */
 883                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 884                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 885                         M_BNE(3 + 4 + 6);
 886                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 887                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 888                         M_BEQ(2 + 3);                                          /* 6 bytes */
 889
 890                         emit_cqto(cd);                                         /* 2 bytes */
 891                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 892
 893                         M_INTMOVE(RDX, d);
 894                         emit_store_dst(jd, iptr, d);
 895                         dst = VAROP(iptr->dst);
 896                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 897                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 898                         break;
 899
 900                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 901                                       /* sx.val.i = constant                             */
 902
 903                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 904                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 905                         M_INTMOVE(s1, REG_ITMP1);
 906                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 907                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 908                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 909                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 910                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 911                         emit_store_dst(jd, iptr, d);
 912                         break;
 913
 914                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 915                                       /* sx.val.l = constant                             */
 916
 917                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 918                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 919                         M_INTMOVE(s1, REG_ITMP1);
 920                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 921                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 922                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 923                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 924                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 925                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 926                         emit_store_dst(jd, iptr, d);
 927                         break;
 928
 929                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 930
 931                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 932                         emit_ishift(jd, SHIFT_SHL, iptr);
 933                         break;
 934
 935                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 936                                       /* sx.val.i = constant                             */
 937
 938                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 939                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 940                         M_INTMOVE(s1, d);
 941                         M_ISLL_IMM(iptr->sx.val.i, d);
 942                         emit_store_dst(jd, iptr, d);
 943                         break;
 944
 945                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 946
 947                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 948                         emit_ishift(jd, SHIFT_SAR, iptr);
 949                         break;
 950
 951                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 952                                       /* sx.val.i = constant                             */
 953
 954                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 955                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 956                         M_INTMOVE(s1, d);
 957                         M_ISRA_IMM(iptr->sx.val.i, d);
 958                         emit_store_dst(jd, iptr, d);
 959                         break;
 960
 961                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 962
 963                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 964                         emit_ishift(jd, SHIFT_SHR, iptr);
 965                         break;
 966
 967                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 968                                       /* sx.val.i = constant                             */
 969
 970                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 971                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 972                         M_INTMOVE(s1, d);
 973                         M_ISRL_IMM(iptr->sx.val.i, d);
 974                         emit_store_dst(jd, iptr, d);
 975                         break;
 976
 977                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 978
 979                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 980                         emit_lshift(jd, SHIFT_SHL, iptr);
 981                         break;
 982
 983         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 984                                           /* sx.val.i = constant                             */
 985
 986                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 987                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 988                         M_INTMOVE(s1, d);
 989                         M_LSLL_IMM(iptr->sx.val.i, d);
 990                         emit_store_dst(jd, iptr, d);
 991                         break;
 992
 993                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 994
 995                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 996                         emit_lshift(jd, SHIFT_SAR, iptr);
 997                         break;
 998
 999                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1000                                       /* sx.val.i = constant                             */
1001
1002                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1003                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1004                         M_INTMOVE(s1, d);
1005                         M_LSRA_IMM(iptr->sx.val.i, d);
1006                         emit_store_dst(jd, iptr, d);
1007                         break;
1008
1009                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1010
1011                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1012                         emit_lshift(jd, SHIFT_SHR, iptr);
1013                         break;
1014
1015                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1016                                       /* sx.val.l = constant                             */
1017
1018                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1019                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1020                         M_INTMOVE(s1, d);
1021                         M_LSRL_IMM(iptr->sx.val.i, d);
1022                         emit_store_dst(jd, iptr, d);
1023                         break;
1024
1025                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1026
1027                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1028                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1029                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1030                         if (s2 == d)
1031                                 M_IAND(s1, d);
1032                         else {
1033                                 M_INTMOVE(s1, d);
1034                                 M_IAND(s2, d);
1035                         }
1036                         emit_store_dst(jd, iptr, d);
1037                         break;
1038
1039                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1040                                       /* sx.val.i = constant                             */
1041
1042                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1043                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1044                         M_INTMOVE(s1, d);
1045                         M_IAND_IMM(iptr->sx.val.i, d);
1046                         emit_store_dst(jd, iptr, d);
1047                         break;
1048
1049                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1050
1051                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1052                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1053                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1054                         if (s2 == d)
1055                                 M_LAND(s1, d);
1056                         else {
1057                                 M_INTMOVE(s1, d);
1058                                 M_LAND(s2, d);
1059                         }
1060                         emit_store_dst(jd, iptr, d);
1061                         break;
1062
1063                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1064                                       /* sx.val.l = constant                             */
1065
1066                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1067                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1068                         M_INTMOVE(s1, d);
1069                         if (IS_IMM32(iptr->sx.val.l))
1070                                 M_LAND_IMM(iptr->sx.val.l, d);
1071                         else {
1072                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1073                                 M_LAND(REG_ITMP2, d);
1074                         }
1075                         emit_store_dst(jd, iptr, d);
1076                         break;
1077
1078                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1079
1080                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1081                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1082                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1083                         if (s2 == d)
1084                                 M_IOR(s1, d);
1085                         else {
1086                                 M_INTMOVE(s1, d);
1087                                 M_IOR(s2, d);
1088                         }
1089                         emit_store_dst(jd, iptr, d);
1090                         break;
1091
1092                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1093                                       /* sx.val.i = constant                             */
1094
1095                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1096                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1097                         M_INTMOVE(s1, d);
1098                         M_IOR_IMM(iptr->sx.val.i, d);
1099                         emit_store_dst(jd, iptr, d);
1100                         break;
1101
1102                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1103
1104                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1105                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1106                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1107                         if (s2 == d)
1108                                 M_LOR(s1, d);
1109                         else {
1110                                 M_INTMOVE(s1, d);
1111                                 M_LOR(s2, d);
1112                         }
1113                         emit_store_dst(jd, iptr, d);
1114                         break;
1115
1116                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1117                                       /* sx.val.l = constant                             */
1118
1119                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1120                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1121                         M_INTMOVE(s1, d);
1122                         if (IS_IMM32(iptr->sx.val.l))
1123                                 M_LOR_IMM(iptr->sx.val.l, d);
1124                         else {
1125                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1126                                 M_LOR(REG_ITMP2, d);
1127                         }
1128                         emit_store_dst(jd, iptr, d);
1129                         break;
1130
1131                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1132
1133                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1134                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1135                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1136                         if (s2 == d)
1137                                 M_IXOR(s1, d);
1138                         else {
1139                                 M_INTMOVE(s1, d);
1140                                 M_IXOR(s2, d);
1141                         }
1142                         emit_store_dst(jd, iptr, d);
1143                         break;
1144
1145                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1146                                       /* sx.val.i = constant                             */
1147
1148                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1149                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1150                         M_INTMOVE(s1, d);
1151                         M_IXOR_IMM(iptr->sx.val.i, d);
1152                         emit_store_dst(jd, iptr, d);
1153                         break;
1154
1155                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1156
1157                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1158                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1159                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1160                         if (s2 == d)
1161                                 M_LXOR(s1, d);
1162                         else {
1163                                 M_INTMOVE(s1, d);
1164                                 M_LXOR(s2, d);
1165                         }
1166                         emit_store_dst(jd, iptr, d);
1167                         break;
1168
1169                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1170                                       /* sx.val.l = constant                             */
1171
1172                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1173                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1174                         M_INTMOVE(s1, d);
1175                         if (IS_IMM32(iptr->sx.val.l))
1176                                 M_LXOR_IMM(iptr->sx.val.l, d);
1177                         else {
1178                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1179                                 M_LXOR(REG_ITMP2, d);
1180                         }
1181                         emit_store_dst(jd, iptr, d);
1182                         break;
1183
1184
1185                 /* floating operations ************************************************/
1186
1187                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1188
1189                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1190                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1191                         disp = dseg_add_s4(cd, 0x80000000);
1192                         M_FLTMOVE(s1, d);
1193                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1194                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1195                         emit_store_dst(jd, iptr, d);
1196                         break;
1197
1198                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1199
1200                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1201                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1202                         disp = dseg_add_s8(cd, 0x8000000000000000);
1203                         M_FLTMOVE(s1, d);
1204                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1205                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1206                         emit_store_dst(jd, iptr, d);
1207                         break;
1208
1209                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1210
1211                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1212                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1213                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1214                         if (s2 == d)
1215                                 M_FADD(s1, d);
1216                         else {
1217                                 M_FLTMOVE(s1, d);
1218                                 M_FADD(s2, d);
1219                         }
1220                         emit_store_dst(jd, iptr, d);
1221                         break;
1222
1223                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1224
1225                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1226                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1227                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1228                         if (s2 == d)
1229                                 M_DADD(s1, d);
1230                         else {
1231                                 M_FLTMOVE(s1, d);
1232                                 M_DADD(s2, d);
1233                         }
1234                         emit_store_dst(jd, iptr, d);
1235                         break;
1236
1237                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1238
1239                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1240                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1241                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1242                         if (s2 == d) {
1243                                 M_FLTMOVE(s2, REG_FTMP2);
1244                                 s2 = REG_FTMP2;
1245                         }
1246                         M_FLTMOVE(s1, d);
1247                         M_FSUB(s2, d);
1248                         emit_store_dst(jd, iptr, d);
1249                         break;
1250
1251                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1252
1253                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1254                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1255                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1256                         if (s2 == d) {
1257                                 M_FLTMOVE(s2, REG_FTMP2);
1258                                 s2 = REG_FTMP2;
1259                         }
1260                         M_FLTMOVE(s1, d);
1261                         M_DSUB(s2, d);
1262                         emit_store_dst(jd, iptr, d);
1263                         break;
1264
1265                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1266
1267                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1268                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1269                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1270                         if (s2 == d)
1271                                 M_FMUL(s1, d);
1272                         else {
1273                                 M_FLTMOVE(s1, d);
1274                                 M_FMUL(s2, d);
1275                         }
1276                         emit_store_dst(jd, iptr, d);
1277                         break;
1278
1279                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1280
1281                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1282                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1283                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1284                         if (s2 == d)
1285                                 M_DMUL(s1, d);
1286                         else {
1287                                 M_FLTMOVE(s1, d);
1288                                 M_DMUL(s2, d);
1289                         }
1290                         emit_store_dst(jd, iptr, d);
1291                         break;
1292
1293                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1294
1295                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1296                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1297                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1298                         if (s2 == d) {
1299                                 M_FLTMOVE(s2, REG_FTMP2);
1300                                 s2 = REG_FTMP2;
1301                         }
1302                         M_FLTMOVE(s1, d);
1303                         M_FDIV(s2, d);
1304                         emit_store_dst(jd, iptr, d);
1305                         break;
1306
1307                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1308
1309                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1310                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1311                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1312                         if (s2 == d) {
1313                                 M_FLTMOVE(s2, REG_FTMP2);
1314                                 s2 = REG_FTMP2;
1315                         }
1316                         M_FLTMOVE(s1, d);
1317                         M_DDIV(s2, d);
1318                         emit_store_dst(jd, iptr, d);
1319                         break;
1320
1321                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1322
1323                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1324                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1325                         M_CVTIF(s1, d);
1326                         emit_store_dst(jd, iptr, d);
1327                         break;
1328
1329                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1330
1331                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1332                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1333                         M_CVTID(s1, d);
1334                         emit_store_dst(jd, iptr, d);
1335                         break;
1336
1337                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1338
1339                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1340                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1341                         M_CVTLF(s1, d);
1342                         emit_store_dst(jd, iptr, d);
1343                         break;
1344
1345                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1346
1347                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1348                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1349                         M_CVTLD(s1, d);
1350                         emit_store_dst(jd, iptr, d);
1351                         break;
1352
1353                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1354
1355                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1356                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1357                         M_CVTFI(s1, d);
1358                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1359                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1360                                 ((REG_RESULT == d) ? 0 : 3);
1361                         M_BNE(disp);
1362                         M_FLTMOVE(s1, REG_FTMP1);
1363                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1364                         M_CALL(REG_ITMP2);
1365                         M_INTMOVE(REG_RESULT, d);
1366                         emit_store_dst(jd, iptr, d);
1367                         break;
1368
1369                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1370
1371                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1372                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1373                         M_CVTDI(s1, d);
1374                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1375                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1376                                 ((REG_RESULT == d) ? 0 : 3);
1377                         M_BNE(disp);
1378                         M_FLTMOVE(s1, REG_FTMP1);
1379                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1380                         M_CALL(REG_ITMP2);
1381                         M_INTMOVE(REG_RESULT, d);
1382                         emit_store_dst(jd, iptr, d);
1383                         break;
1384
1385                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1386
1387                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1388                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1389                         M_CVTFL(s1, d);
1390                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1391                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1392                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1393                                 ((REG_RESULT == d) ? 0 : 3);
1394                         M_BNE(disp);
1395                         M_FLTMOVE(s1, REG_FTMP1);
1396                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1397                         M_CALL(REG_ITMP2);
1398                         M_INTMOVE(REG_RESULT, d);
1399                         emit_store_dst(jd, iptr, d);
1400                         break;
1401
1402                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1403
1404                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1405                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1406                         M_CVTDL(s1, d);
1407                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1408                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1409                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1410                                 ((REG_RESULT == d) ? 0 : 3);
1411                         M_BNE(disp);
1412                         M_FLTMOVE(s1, REG_FTMP1);
1413                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1414                         M_CALL(REG_ITMP2);
1415                         M_INTMOVE(REG_RESULT, d);
1416                         emit_store_dst(jd, iptr, d);
1417                         break;
1418
1419                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1420
1421                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1422                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1423                         M_CVTFD(s1, d);
1424                         emit_store_dst(jd, iptr, d);
1425                         break;
1426
1427                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1428
1429                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1430                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1431                         M_CVTDF(s1, d);
1432                         emit_store_dst(jd, iptr, d);
1433                         break;
1434
1435                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1436                                           /* == => 0, < => 1, > => -1 */
1437
1438                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1439                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1440                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1441                         M_CLR(d);
1442                         M_MOV_IMM(1, REG_ITMP1);
1443                         M_MOV_IMM(-1, REG_ITMP2);
1444                         emit_ucomiss_reg_reg(cd, s1, s2);
1445                         M_CMOVULT(REG_ITMP1, d);
1446                         M_CMOVUGT(REG_ITMP2, d);
1447                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1448                         emit_store_dst(jd, iptr, d);
1449                         break;
1450
1451                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1452                                           /* == => 0, < => 1, > => -1 */
1453
1454                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1455                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1456                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1457                         M_CLR(d);
1458                         M_MOV_IMM(1, REG_ITMP1);
1459                         M_MOV_IMM(-1, REG_ITMP2);
1460                         emit_ucomiss_reg_reg(cd, s1, s2);
1461                         M_CMOVULT(REG_ITMP1, d);
1462                         M_CMOVUGT(REG_ITMP2, d);
1463                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1464                         emit_store_dst(jd, iptr, d);
1465                         break;
1466
1467                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1468                                           /* == => 0, < => 1, > => -1 */
1469
1470                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1471                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1472                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1473                         M_CLR(d);
1474                         M_MOV_IMM(1, REG_ITMP1);
1475                         M_MOV_IMM(-1, REG_ITMP2);
1476                         emit_ucomisd_reg_reg(cd, s1, s2);
1477                         M_CMOVULT(REG_ITMP1, d);
1478                         M_CMOVUGT(REG_ITMP2, d);
1479                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1480                         emit_store_dst(jd, iptr, d);
1481                         break;
1482
1483                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1484                                           /* == => 0, < => 1, > => -1 */
1485
1486                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1487                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1488                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1489                         M_CLR(d);
1490                         M_MOV_IMM(1, REG_ITMP1);
1491                         M_MOV_IMM(-1, REG_ITMP2);
1492                         emit_ucomisd_reg_reg(cd, s1, s2);
1493                         M_CMOVULT(REG_ITMP1, d);
1494                         M_CMOVUGT(REG_ITMP2, d);
1495                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1496                         emit_store_dst(jd, iptr, d);
1497                         break;
1498
1499
1500                 /* memory operations **************************************************/
1501
1502                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1503
1504                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1505                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1506                         /* implicit null-pointer check */
1507                         M_ILD(d, s1, OFFSET(java_array_t, size));
1508                         emit_store_dst(jd, iptr, d);
1509                         break;
1510
1511                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1512
1513                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1514                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1515                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1516                         /* implicit null-pointer check */
1517                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1518                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1519                         emit_store_dst(jd, iptr, d);
1520                         break;
1521
1522                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1523
1524                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1525                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1526                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1527                         /* implicit null-pointer check */
1528                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1529                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1530                         emit_store_dst(jd, iptr, d);
1531                         break;
1532
1533                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1534
1535                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1536                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1537                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1538                         /* implicit null-pointer check */
1539                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1540                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1541                         emit_store_dst(jd, iptr, d);
1542                         break;
1543
1544                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1545
1546                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1547                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1548                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1549                         /* implicit null-pointer check */
1550                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1551                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1552                         emit_store_dst(jd, iptr, d);
1553                         break;
1554
1555                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1556
1557                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1558                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1559                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1560                         /* implicit null-pointer check */
1561                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1562                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1563                         emit_store_dst(jd, iptr, d);
1564                         break;
1565
1566                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1567
1568                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1569                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1570                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1571                         /* implicit null-pointer check */
1572                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1573                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1574                         emit_store_dst(jd, iptr, d);
1575                         break;
1576
1577                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1578
1579                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1580                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1581                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1582                         /* implicit null-pointer check */
1583                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1584                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1585                         emit_store_dst(jd, iptr, d);
1586                         break;
1587
1588                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1589
1590                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1591                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1592                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1593                         /* implicit null-pointer check */
1594                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1595                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1596                         emit_store_dst(jd, iptr, d);
1597                         break;
1598
1599
1600                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1601
1602                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1603                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1604                         /* implicit null-pointer check */
1605                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1606                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1607                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1608                         break;
1609
1610                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1611
1612                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1613                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1614                         /* implicit null-pointer check */
1615                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1616                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1617                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1618                         break;
1619
1620                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1621
1622                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1623                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1624                         /* implicit null-pointer check */
1625                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1626                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1627                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1628                         break;
1629
1630                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1631
1632                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1633                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1634                         /* implicit null-pointer check */
1635                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1636                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1637                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1638                         break;
1639
1640                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1641
1642                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1643                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1644                         /* implicit null-pointer check */
1645                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1646                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1647                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1648                         break;
1649
1650                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1651
1652                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1653                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1654                         /* implicit null-pointer check */
1655                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1656                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1657                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1658                         break;
1659
1660                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1661
1662                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1663                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1664                         /* implicit null-pointer check */
1665                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1666                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1667                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1668                         break;
1669
1670                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1671
1672                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1673                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1674                         /* implicit null-pointer check */
1675                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1676                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1677
1678                         M_MOV(s1, REG_A0);
1679                         M_MOV(s3, REG_A1);
1680                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
1681                         M_CALL(REG_ITMP1);
1682                         emit_arraystore_check(cd, iptr);
1683
1684                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1685                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1686                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1687                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1688                         break;
1689
1690
1691                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1692
1693                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1694                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1695                         /* implicit null-pointer check */
1696                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1697                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1698                         break;
1699
1700                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1701
1702                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1703                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1704                         /* implicit null-pointer check */
1705                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1706                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1707                         break;
1708
1709                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1710
1711                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1712                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1713                         /* implicit null-pointer check */
1714                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1715                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1716                         break;
1717
1718                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1719
1720                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1721                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1722                         /* implicit null-pointer check */
1723                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1724                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1725                         break;
1726
1727                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1728
1729                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1730                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1731                         /* implicit null-pointer check */
1732                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1733
1734                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1735                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1736                         }
1737                         else {
1738                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1739                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1740                         }
1741                         break;
1742
1743                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1744
1745                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1746                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1747                         /* implicit null-pointer check */
1748                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1749                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1750                         break;
1751
1752
1753                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1754
1755                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1756                                 uf        = iptr->sx.s23.s3.uf;
1757                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1758                                 disp      = dseg_add_unique_address(cd, uf);
1759
1760 /*                              PROFILE_CYCLE_STOP; */
1761
1762                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1763
1764 /*                              PROFILE_CYCLE_START; */
1765                         }
1766                         else {
1767                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1768                                 fieldtype = fi->type;
1769                                 disp      = dseg_add_address(cd, fi->value);
1770
1771                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1772                                         PROFILE_CYCLE_STOP;
1773
1774                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1775                                                                                   fi->class, 0);
1776
1777                                         PROFILE_CYCLE_START;
1778                                 }
1779                         }
1780
1781                         /* This approach is much faster than moving the field
1782                            address inline into a register. */
1783
1784                         M_ALD(REG_ITMP1, RIP, disp);
1785
1786                         switch (fieldtype) {
1787                         case TYPE_INT:
1788                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1789                                 M_ILD(d, REG_ITMP1, 0);
1790                                 break;
1791                         case TYPE_LNG:
1792                         case TYPE_ADR:
1793                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1794                                 M_LLD(d, REG_ITMP1, 0);
1795                                 break;
1796                         case TYPE_FLT:
1797                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1798                                 M_FLD(d, REG_ITMP1, 0);
1799                                 break;
1800                         case TYPE_DBL:
1801                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1802                                 M_DLD(d, REG_ITMP1, 0);
1803                                 break;
1804                         }
1805                         emit_store_dst(jd, iptr, d);
1806                         break;
1807
1808                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1809
1810                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1811                                 uf        = iptr->sx.s23.s3.uf;
1812                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1813                                 disp      = dseg_add_unique_address(cd, uf);
1814
1815 /*                              PROFILE_CYCLE_STOP; */
1816
1817                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1818
1819 /*                              PROFILE_CYCLE_START; */
1820                         }
1821                         else {
1822                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1823                                 fieldtype = fi->type;
1824                                 disp      = dseg_add_address(cd, fi->value);
1825
1826                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1827                                         PROFILE_CYCLE_STOP;
1828
1829                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1830                                                                                   fi->class, 0);
1831
1832                                         PROFILE_CYCLE_START;
1833                                 }
1834                         }
1835
1836                         /* This approach is much faster than moving the field
1837                            address inline into a register. */
1838
1839                         M_ALD(REG_ITMP1, RIP, disp);
1840
1841                         switch (fieldtype) {
1842                         case TYPE_INT:
1843                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1844                                 M_IST(s1, REG_ITMP1, 0);
1845                                 break;
1846                         case TYPE_LNG:
1847                         case TYPE_ADR:
1848                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1849                                 M_LST(s1, REG_ITMP1, 0);
1850                                 break;
1851                         case TYPE_FLT:
1852                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1853                                 M_FST(s1, REG_ITMP1, 0);
1854                                 break;
1855                         case TYPE_DBL:
1856                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1857                                 M_DST(s1, REG_ITMP1, 0);
1858                                 break;
1859                         }
1860                         break;
1861
1862                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1863                                           /* val = value (in current instruction)     */
1864                                           /* following NOP)                           */
1865
1866                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1867                                 uf        = iptr->sx.s23.s3.uf;
1868                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1869                                 disp      = dseg_add_unique_address(cd, uf);
1870
1871 /*                              PROFILE_CYCLE_STOP; */
1872
1873                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1874
1875 /*                              PROFILE_CYCLE_START; */
1876                         }
1877                         else {
1878                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1879                                 fieldtype = fi->type;
1880                                 disp      = dseg_add_address(cd, fi->value);
1881
1882                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1883                                         PROFILE_CYCLE_STOP;
1884
1885                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1886                                                                                   fi->class, 0);
1887
1888                                         PROFILE_CYCLE_START;
1889                                 }
1890                         }
1891
1892                         /* This approach is much faster than moving the field
1893                            address inline into a register. */
1894
1895                         M_ALD(REG_ITMP1, RIP, disp);
1896
1897                         switch (fieldtype) {
1898                         case TYPE_INT:
1899                         case TYPE_FLT:
1900                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1901                                 break;
1902                         case TYPE_LNG:
1903                         case TYPE_ADR:
1904                         case TYPE_DBL:
1905                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1906                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1907                                 else {
1908                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1909                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
1910                                 }
1911                                 break;
1912                         }
1913                         break;
1914
1915                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1916
1917                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1918
1919                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1920                                 uf        = iptr->sx.s23.s3.uf;
1921                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1922                                 disp      = 0;
1923
1924 /*                              PROFILE_CYCLE_STOP; */
1925
1926                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1927
1928 /*                              PROFILE_CYCLE_START; */
1929                         }
1930                         else {
1931                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1932                                 fieldtype = fi->type;
1933                                 disp      = fi->offset;
1934                         }
1935
1936                         /* implicit null-pointer check */
1937                         switch (fieldtype) {
1938                         case TYPE_INT:
1939                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1940                                 M_ILD32(d, s1, disp);
1941                                 break;
1942                         case TYPE_LNG:
1943                         case TYPE_ADR:
1944                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1945                                 M_LLD32(d, s1, disp);
1946                                 break;
1947                         case TYPE_FLT:
1948                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1949                                 M_FLD32(d, s1, disp);
1950                                 break;
1951                         case TYPE_DBL:
1952                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1953                                 M_DLD32(d, s1, disp);
1954                                 break;
1955                         }
1956                         emit_store_dst(jd, iptr, d);
1957                         break;
1958
1959                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1960
1961                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1962                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1963
1964                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1965                                 uf        = iptr->sx.s23.s3.uf;
1966                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1967                                 disp      = 0;
1968
1969 /*                              PROFILE_CYCLE_STOP; */
1970
1971                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1972
1973 /*                              PROFILE_CYCLE_START; */
1974                         }
1975                         else {
1976                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1977                                 fieldtype = fi->type;
1978                                 disp      = fi->offset;
1979                         }
1980
1981                         /* implicit null-pointer check */
1982                         switch (fieldtype) {
1983                         case TYPE_INT:
1984                                 M_IST32(s2, s1, disp);
1985                                 break;
1986                         case TYPE_LNG:
1987                         case TYPE_ADR:
1988                                 M_LST32(s2, s1, disp);
1989                                 break;
1990                         case TYPE_FLT:
1991                                 M_FST32(s2, s1, disp);
1992                                 break;
1993                         case TYPE_DBL:
1994                                 M_DST32(s2, s1, disp);
1995                                 break;
1996                         }
1997                         break;
1998
1999                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2000                                           /* val = value (in current instruction)     */
2001                                           /* following NOP)                           */
2002
2003                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2004
2005                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2006                                 uf        = iptr->sx.s23.s3.uf;
2007                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2008                                 disp      = 0;
2009
2010 /*                              PROFILE_CYCLE_STOP; */
2011
2012                                 patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
2013
2014 /*                              PROFILE_CYCLE_START; */
2015                         }
2016                         else {
2017                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2018                                 fieldtype = fi->type;
2019                                 disp      = fi->offset;
2020                         }
2021
2022                         /* implicit null-pointer check */
2023                         switch (fieldtype) {
2024                         case TYPE_INT:
2025                         case TYPE_FLT:
2026                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2027                                 break;
2028                         case TYPE_LNG:
2029                         case TYPE_ADR:
2030                         case TYPE_DBL:
2031                                 /* XXX why no check for IS_IMM32? */
2032                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2033                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2034                                 break;
2035                         }
2036                         break;
2037
2038
2039                 /* branch operations **************************************************/
2040
2041                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2042
2043                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2044                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2045
2046                         PROFILE_CYCLE_STOP;
2047
2048 #ifdef ENABLE_VERIFIER
2049                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2050                                 uc = iptr->sx.s23.s2.uc;
2051
2052                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2053                         }
2054 #endif /* ENABLE_VERIFIER */
2055
2056                         M_CALL_IMM(0);                            /* passing exception pc */
2057                         M_POP(REG_ITMP2_XPC);
2058
2059                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2060                         M_JMP(REG_ITMP3);
2061                         break;
2062
2063                 case ICMD_GOTO:         /* ... ==> ...                                */
2064                 case ICMD_RET:
2065
2066                         emit_br(cd, iptr->dst.block);
2067                         ALIGNCODENOP;
2068                         break;
2069
2070                 case ICMD_JSR:          /* ... ==> ...                                */
2071
2072                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2073                         ALIGNCODENOP;
2074                         break;
2075
2076                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2077                 case ICMD_IFNONNULL:
2078
2079                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2080                         M_TEST(s1);
2081                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2082                         break;
2083
2084                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2085                 case ICMD_IFLT:
2086                 case ICMD_IFLE:
2087                 case ICMD_IFNE:
2088                 case ICMD_IFGT:
2089                 case ICMD_IFGE:
2090
2091                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2092                         M_ICMP_IMM(iptr->sx.val.i, s1);
2093                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2094                         break;
2095
2096                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2097                 case ICMD_IF_LNE:
2098                 case ICMD_IF_LLT:
2099                 case ICMD_IF_LGE:
2100                 case ICMD_IF_LGT:
2101                 case ICMD_IF_LLE:
2102
2103                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2104                         if (IS_IMM32(iptr->sx.val.l))
2105                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2106                         else {
2107                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2108                                 M_LCMP(REG_ITMP2, s1);
2109                         }
2110                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2111                         break;
2112
2113                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2114                 case ICMD_IF_ICMPNE:
2115                 case ICMD_IF_ICMPLT:
2116                 case ICMD_IF_ICMPGE:
2117                 case ICMD_IF_ICMPGT:
2118                 case ICMD_IF_ICMPLE:
2119
2120                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2121                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2122                         M_ICMP(s2, s1);
2123                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2124                         break;
2125
2126                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2127                 case ICMD_IF_ACMPNE:
2128
2129                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2130                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2131                         M_LCMP(s2, s1);
2132                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2133                         break;
2134
2135                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2136                 case ICMD_IF_LCMPNE:
2137                 case ICMD_IF_LCMPLT:
2138                 case ICMD_IF_LCMPGE:
2139                 case ICMD_IF_LCMPGT:
2140                 case ICMD_IF_LCMPLE:
2141
2142                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2143                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2144                         M_LCMP(s2, s1);
2145                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2146                         break;
2147
2148                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2149                 case ICMD_LRETURN:
2150
2151                         REPLACEMENT_POINT_RETURN(cd, iptr);
2152                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2153                         M_INTMOVE(s1, REG_RESULT);
2154                         goto nowperformreturn;
2155
2156                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2157
2158                         REPLACEMENT_POINT_RETURN(cd, iptr);
2159                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2160                         M_INTMOVE(s1, REG_RESULT);
2161
2162 #ifdef ENABLE_VERIFIER
2163                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2164                                 uc = iptr->sx.s23.s2.uc;
2165
2166                                 PROFILE_CYCLE_STOP;
2167
2168                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2169
2170                                 PROFILE_CYCLE_START;
2171                         }
2172 #endif /* ENABLE_VERIFIER */
2173                         goto nowperformreturn;
2174
2175                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2176                 case ICMD_DRETURN:
2177
2178                         REPLACEMENT_POINT_RETURN(cd, iptr);
2179                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2180                         M_FLTMOVE(s1, REG_FRESULT);
2181                         goto nowperformreturn;
2182
2183                 case ICMD_RETURN:      /* ...  ==> ...                                */
2184
2185                         REPLACEMENT_POINT_RETURN(cd, iptr);
2186
2187 nowperformreturn:
2188                         {
2189                         s4 i, p;
2190
2191                         p = cd->stackframesize;
2192
2193 #if !defined(NDEBUG)
2194                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2195                                 emit_verbosecall_exit(jd);
2196 #endif /* !defined(NDEBUG) */
2197
2198 #if defined(ENABLE_THREADS)
2199                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2200                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2201
2202                                 /* we need to save the proper return value */
2203                                 switch (iptr->opc) {
2204                                 case ICMD_IRETURN:
2205                                 case ICMD_ARETURN:
2206                                 case ICMD_LRETURN:
2207                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2208                                         break;
2209                                 case ICMD_FRETURN:
2210                                 case ICMD_DRETURN:
2211                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2212                                         break;
2213                                 }
2214
2215                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2216                                 M_CALL(REG_ITMP1);
2217
2218                                 /* and now restore the proper return value */
2219                                 switch (iptr->opc) {
2220                                 case ICMD_IRETURN:
2221                                 case ICMD_ARETURN:
2222                                 case ICMD_LRETURN:
2223                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2224                                         break;
2225                                 case ICMD_FRETURN:
2226                                 case ICMD_DRETURN:
2227                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2228                                         break;
2229                                 }
2230                         }
2231 #endif
2232
2233                         /* restore saved registers */
2234
2235                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2236                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2237                         }
2238                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2239                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2240                         }
2241
2242                         /* deallocate stack */
2243
2244                         if (cd->stackframesize)
2245                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2246
2247                         /* generate method profiling code */
2248
2249                         PROFILE_CYCLE_STOP;
2250
2251                         M_RET;
2252                         }
2253                         break;
2254
2255
2256                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2257                         {
2258                                 s4 i, l;
2259                                 branch_target_t *table;
2260
2261                                 table = iptr->dst.table;
2262
2263                                 l = iptr->sx.s23.s2.tablelow;
2264                                 i = iptr->sx.s23.s3.tablehigh;
2265
2266                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2267                                 M_INTMOVE(s1, REG_ITMP1);
2268
2269                                 if (l != 0)
2270                                         M_ISUB_IMM(l, REG_ITMP1);
2271
2272                                 /* number of targets */
2273                                 i = i - l + 1;
2274
2275                 /* range check */
2276
2277                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2278                                 emit_bugt(cd, table[0].block);
2279
2280                                 /* build jump table top down and use address of lowest entry */
2281
2282                                 table += i;
2283
2284                                 while (--i >= 0) {
2285                                         dseg_add_target(cd, table->block);
2286                                         --table;
2287                                 }
2288
2289                                 /* length of dataseg after last dseg_add_target is used
2290                                    by load */
2291
2292                                 M_MOV_IMM(0, REG_ITMP2);
2293                                 dseg_adddata(cd);
2294                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2295                                 M_JMP(REG_ITMP1);
2296                         }
2297                         break;
2298
2299
2300                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2301                         {
2302                                 s4 i;
2303                                 lookup_target_t *lookup;
2304
2305                                 lookup = iptr->dst.lookup;
2306
2307                                 i = iptr->sx.s23.s2.lookupcount;
2308
2309                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2310                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2311
2312                                 while (--i >= 0) {
2313                                         M_ICMP_IMM(lookup->value, s1);
2314                                         emit_beq(cd, lookup->target.block);
2315                                         lookup++;
2316                                 }
2317
2318                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2319                                 ALIGNCODENOP;
2320                         }
2321                         break;
2322
2323
2324                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2325
2326                         REPLACEMENT_POINT_FORGC_BUILTIN(cd, iptr);
2327
2328                         bte = iptr->sx.s23.s3.bte;
2329                         md  = bte->md;
2330                         goto gen_method;
2331
2332                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2333
2334                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2335                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2336                 case ICMD_INVOKEINTERFACE:
2337
2338                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2339
2340                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2341                                 lm = NULL;
2342                                 um = iptr->sx.s23.s3.um;
2343                                 md = um->methodref->parseddesc.md;
2344                         }
2345                         else {
2346                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2347                                 um = NULL;
2348                                 md = lm->parseddesc;
2349                         }
2350
2351 gen_method:
2352                         s3 = md->paramcount;
2353
2354                         MCODECHECK((20 * s3) + 128);
2355
2356                         /* copy arguments to registers or stack location */
2357
2358                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2359                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2360                                 d   = md->params[s3].regoff;
2361
2362                                 /* already preallocated (ARGVAR)? */
2363
2364                                 if (var->flags & PREALLOC)
2365                                         continue;
2366
2367                                 if (IS_INT_LNG_TYPE(var->type)) {
2368                                         if (!md->params[s3].inmemory) {
2369                                                 s1 = emit_load(jd, iptr, var, d);
2370                                                 M_INTMOVE(s1, d);
2371                                         }
2372                                         else {
2373                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2374                                                 M_LST(s1, REG_SP, d);
2375                                         }
2376                                 }
2377                                 else {
2378                                         if (!md->params[s3].inmemory) {
2379                                                 s1 = emit_load(jd, iptr, var, d);
2380                                                 M_FLTMOVE(s1, d);
2381                                         }
2382                                         else {
2383                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2384
2385                                                 if (IS_2_WORD_TYPE(var->type))
2386                                                         M_DST(s1, REG_SP, d);
2387                                                 else
2388                                                         M_FST(s1, REG_SP, d);
2389                                         }
2390                                 }
2391                         }
2392
2393                         /* generate method profiling code */
2394
2395                         PROFILE_CYCLE_STOP;
2396
2397                         switch (iptr->opc) {
2398                         case ICMD_BUILTIN:
2399                                 if (bte->stub == NULL) {
2400                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2401                                 } else {
2402                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2403                                 }
2404                                 M_CALL(REG_ITMP1);
2405
2406                                 emit_exception_check(cd, iptr);
2407                                 break;
2408
2409                         case ICMD_INVOKESPECIAL:
2410                                 emit_nullpointer_check(cd, iptr, REG_A0);
2411                                 /* fall through */
2412
2413                         case ICMD_INVOKESTATIC:
2414                                 if (lm == NULL) {
2415                                         disp = dseg_add_unique_address(cd, um);
2416
2417                                         patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
2418                                                                                   um, disp);
2419                                 }
2420                                 else {
2421                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2422                                 }
2423
2424                                 M_ALD(REG_ITMP2, RIP, disp);
2425                                 M_CALL(REG_ITMP2);
2426                                 break;
2427
2428                         case ICMD_INVOKEVIRTUAL:
2429                                 if (lm == NULL) {
2430                                         patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
2431
2432                                         s1 = 0;
2433                                 }
2434                                 else {
2435                                         s1 = OFFSET(vftbl_t, table[0]) +
2436                                                 sizeof(methodptr) * lm->vftblindex;
2437                                 }
2438
2439                                 /* implicit null-pointer check */
2440                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2441                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2442                                 M_CALL(REG_ITMP3);
2443                                 break;
2444
2445                         case ICMD_INVOKEINTERFACE:
2446                                 if (lm == NULL) {
2447                                         patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
2448
2449                                         s1 = 0;
2450                                         s2 = 0;
2451                                 }
2452                                 else {
2453                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2454                                                 sizeof(methodptr) * lm->class->index;
2455
2456                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2457                                 }
2458
2459                                 /* implicit null-pointer check */
2460                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2461                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2462                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2463                                 M_CALL(REG_ITMP3);
2464                                 break;
2465                         }
2466
2467                         /* generate method profiling code */
2468
2469                         PROFILE_CYCLE_START;
2470
2471                         /* store size of call code in replacement point */
2472
2473                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2474                         REPLACEMENT_POINT_FORGC_BUILTIN_RETURN(cd, iptr);
2475
2476                         /* store return value */
2477
2478                         switch (md->returntype.type) {
2479                         case TYPE_INT:
2480                         case TYPE_LNG:
2481                         case TYPE_ADR:
2482                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2483                                 M_INTMOVE(REG_RESULT, s1);
2484                                 emit_store_dst(jd, iptr, s1);
2485                                 break;
2486                         case TYPE_FLT:
2487                         case TYPE_DBL:
2488                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2489                                 M_FLTMOVE(REG_FRESULT, s1);
2490                                 emit_store_dst(jd, iptr, s1);
2491                                 break;
2492                         default:
2493                                 /* TYPE_VOID */
2494                                 break;
2495                         }
2496                         break;
2497
2498
2499                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2500
2501                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2502                                 /* object type cast-check */
2503
2504                                 classinfo *super;
2505                                 s4         superindex;
2506
2507                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2508                                         super      = NULL;
2509                                         superindex = 0;
2510                                 }
2511                                 else {
2512                                         super      = iptr->sx.s23.s3.c.cls;
2513                                         superindex = super->index;
2514                                 }
2515
2516                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2517                                         CODEGEN_CRITICAL_SECTION_NEW;
2518
2519                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2520
2521                                 /* if class is not resolved, check which code to call */
2522
2523                                 if (super == NULL) {
2524                                         M_TEST(s1);
2525                                         emit_label_beq(cd, BRANCH_LABEL_1);
2526
2527                                         patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2528                                                                                   iptr->sx.s23.s3.c.ref, 0);
2529
2530                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2531                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2532                                         emit_label_beq(cd, BRANCH_LABEL_2);
2533                                 }
2534
2535                                 /* interface checkcast code */
2536
2537                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2538                                         if (super != NULL) {
2539                                                 M_TEST(s1);
2540                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2541                                         }
2542
2543                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2544
2545                                         if (super == NULL) {
2546                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
2547                                                                                           iptr->sx.s23.s3.c.ref,
2548                                                                                           0);
2549                                         }
2550
2551                                         M_ILD32(REG_ITMP3,
2552                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2553                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2554                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2555
2556                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2557                                                         OFFSET(vftbl_t, interfacetable[0]) -
2558                                                         superindex * sizeof(methodptr*));
2559                                         M_TEST(REG_ITMP3);
2560                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2561
2562                                         if (super == NULL)
2563                                                 emit_label_br(cd, BRANCH_LABEL_4);
2564                                         else
2565                                                 emit_label(cd, BRANCH_LABEL_3);
2566                                 }
2567
2568                                 /* class checkcast code */
2569
2570                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2571                                         if (super == NULL) {
2572                                                 emit_label(cd, BRANCH_LABEL_2);
2573
2574                                                 cr   = iptr->sx.s23.s3.c.ref;
2575                                                 disp = dseg_add_unique_address(cd, cr);
2576
2577                                                 patcher_add_patch_ref(jd,
2578                                                                                           PATCHER_resolve_classref_to_vftbl,
2579                                                                                           cr, disp);
2580                                         }
2581                                         else {
2582                                                 M_TEST(s1);
2583                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2584
2585                                                 disp = dseg_add_address(cd, super->vftbl);
2586                                         }
2587
2588                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2589                                         M_ALD(REG_ITMP3, RIP, disp);
2590
2591                                         CODEGEN_CRITICAL_SECTION_START;
2592
2593                                         M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2594
2595                                         /*                                      if (s1 != REG_ITMP1) { */
2596                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2597                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2598                                         /*                                                                                              REG_ITMP1); */
2599                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2600                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2601                                         /*                                                                                              REG_ITMP3); */
2602                                         /*  #if defined(ENABLE_THREADS) */
2603                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2604                                         /*  #endif */
2605                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2606
2607                                         /*                                      } else { */
2608
2609                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
2610                                         M_ISUB(REG_ITMP3, REG_ITMP2);
2611                                         M_ALD(REG_ITMP3, RIP, disp);
2612                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2613                                         /*                                      } */
2614
2615                                         CODEGEN_CRITICAL_SECTION_END;
2616
2617                                         M_ICMP(REG_ITMP3, REG_ITMP2);
2618                                         emit_classcast_check(cd, iptr, BRANCH_UGT, REG_ITMP3, s1);
2619
2620                                         if (super != NULL)
2621                                                 emit_label(cd, BRANCH_LABEL_5);
2622                                 }
2623
2624                                 if (super == NULL) {
2625                                         emit_label(cd, BRANCH_LABEL_1);
2626                                         emit_label(cd, BRANCH_LABEL_4);
2627                                 }
2628
2629                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2630                         }
2631                         else {
2632                                 /* array type cast-check */
2633
2634                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2635                                 M_INTMOVE(s1, REG_A0);
2636
2637                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2638                                         cr   = iptr->sx.s23.s3.c.ref;
2639                                         disp = dseg_add_unique_address(cd, cr);
2640
2641                                         patcher_add_patch_ref(jd,
2642                                                                                   PATCHER_resolve_classref_to_classinfo,
2643                                                                                   cr, disp);
2644                                 }
2645                                 else {
2646                                         disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2647                                 }
2648
2649                                 M_ALD(REG_A1, RIP, disp);
2650                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2651                                 M_CALL(REG_ITMP1);
2652
2653                                 /* s1 may have been destroyed over the function call */
2654                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2655                                 M_TEST(REG_RESULT);
2656                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2657
2658                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2659                         }
2660
2661                         M_INTMOVE(s1, d);
2662                         emit_store_dst(jd, iptr, d);
2663                         break;
2664
2665                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2666
2667                         {
2668                         classinfo *super;
2669                         s4         superindex;
2670
2671                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2672                                 super      = NULL;
2673                                 superindex = 0;
2674                         }
2675                         else {
2676                                 super      = iptr->sx.s23.s3.c.cls;
2677                                 superindex = super->index;
2678                         }
2679
2680                         if ((super == NULL) || !(super->flags & ACC_INTERFACE))
2681                                 CODEGEN_CRITICAL_SECTION_NEW;
2682
2683                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2684                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2685
2686                         if (s1 == d) {
2687                                 M_INTMOVE(s1, REG_ITMP1);
2688                                 s1 = REG_ITMP1;
2689                         }
2690
2691                         M_CLR(d);
2692
2693                         /* if class is not resolved, check which code to call */
2694
2695                         if (super == NULL) {
2696                                 M_TEST(s1);
2697                                 emit_label_beq(cd, BRANCH_LABEL_1);
2698
2699                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2700                                                                           iptr->sx.s23.s3.c.ref, 0);
2701
2702                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2703                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2704                                 emit_label_beq(cd, BRANCH_LABEL_2);
2705                         }
2706
2707                         /* interface instanceof code */
2708
2709                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2710                                 if (super != NULL) {
2711                                         M_TEST(s1);
2712                                         emit_label_beq(cd, BRANCH_LABEL_3);
2713                                 }
2714
2715                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2716
2717                                 if (super == NULL) {
2718                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
2719                                                                                   iptr->sx.s23.s3.c.ref, 0);
2720                                 }
2721
2722                                 M_ILD32(REG_ITMP3,
2723                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2724                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2725
2726                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2727
2728                                 M_BLE(a);
2729                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2730                                                 OFFSET(vftbl_t, interfacetable[0]) -
2731                                                 superindex * sizeof(methodptr*));
2732                                 M_TEST(REG_ITMP1);
2733                                 M_SETNE(d);
2734
2735                                 if (super == NULL)
2736                                         emit_label_br(cd, BRANCH_LABEL_4);
2737                                 else
2738                                         emit_label(cd, BRANCH_LABEL_3);
2739                         }
2740
2741                         /* class instanceof code */
2742
2743                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2744                                 if (super == NULL) {
2745                                         emit_label(cd, BRANCH_LABEL_2);
2746
2747                                         cr   = iptr->sx.s23.s3.c.ref;
2748                                         disp = dseg_add_unique_address(cd, cr);
2749
2750                                         patcher_add_patch_ref(jd,
2751                                                                                   PATCHER_resolve_classref_to_vftbl,
2752                                                                                   cr, disp);
2753                                 }
2754                                 else {
2755                                         M_TEST(s1);
2756                                         emit_label_beq(cd, BRANCH_LABEL_5);
2757
2758                                         disp = dseg_add_address(cd, super->vftbl);
2759                                 }
2760
2761                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2762                                 M_ALD(REG_ITMP2, RIP, disp);
2763
2764                                 CODEGEN_CRITICAL_SECTION_START;
2765
2766                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
2767                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2768                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2769
2770                                 CODEGEN_CRITICAL_SECTION_END;
2771
2772                                 M_ISUB(REG_ITMP2, REG_ITMP1);
2773                                 M_CLR(d); /* may be REG_ITMP2 */
2774                                 M_ICMP(REG_ITMP3, REG_ITMP1);
2775                                 M_SETULE(d);
2776
2777                                 if (super != NULL)
2778                                         emit_label(cd, BRANCH_LABEL_5);
2779                         }
2780
2781                         if (super == NULL) {
2782                                 emit_label(cd, BRANCH_LABEL_1);
2783                                 emit_label(cd, BRANCH_LABEL_4);
2784                         }
2785
2786                         emit_store_dst(jd, iptr, d);
2787                         }
2788                         break;
2789
2790                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2791
2792                         /* check for negative sizes and copy sizes to stack if necessary  */
2793
2794                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2795
2796                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2797
2798                                 /* copy SAVEDVAR sizes to stack */
2799                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2800
2801                                 /* Already Preallocated? */
2802                                 if (!(var->flags & PREALLOC)) {
2803                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2804                                         M_LST(s2, REG_SP, s1 * 8);
2805                                 }
2806                         }
2807
2808                         /* a0 = dimension count */
2809
2810                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2811
2812                         /* is a patcher function set? */
2813
2814                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2815                                 cr   = iptr->sx.s23.s3.c.ref;
2816                                 disp = dseg_add_unique_address(cd, cr);
2817
2818                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
2819                                                                           cr, disp);
2820                         }
2821                         else {
2822                                 disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2823                         }
2824
2825                         /* a1 = classinfo */
2826
2827                         M_ALD(REG_A1, RIP, disp);
2828
2829                         /* a2 = pointer to dimensions = stack pointer */
2830
2831                         M_MOV(REG_SP, REG_A2);
2832
2833                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2834                         M_CALL(REG_ITMP1);
2835
2836                         /* check for exception before result assignment */
2837
2838                         emit_exception_check(cd, iptr);
2839
2840                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2841                         M_INTMOVE(REG_RESULT, s1);
2842                         emit_store_dst(jd, iptr, s1);
2843                         break;
2844
2845                 default:
2846                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2847                                                                                    iptr->opc);
2848                         return false;
2849         } /* switch */
2850
2851         } /* for instruction */
2852
2853         MCODECHECK(512); /* XXX require a lower number? */
2854
2855         /* At the end of a basic block we may have to append some nops,
2856            because the patcher stub calling code might be longer than the
2857            actual instruction. So codepatching does not change the
2858            following block unintentionally. */
2859
2860         if (cd->mcodeptr < cd->lastmcodeptr) {
2861                 while (cd->mcodeptr < cd->lastmcodeptr) {
2862                         M_NOP;
2863                 }
2864         }
2865
2866         } /* if (bptr -> flags >= BBREACHED) */
2867         } /* for basic block */
2868
2869         dseg_createlinenumbertable(cd);
2870
2871         /* Generate patcher traps. */
2872
2873         emit_patcher_traps(jd);
2874
2875         /* everything's ok */
2876
2877         return true;
2878 }
2879
2880
2881 /* codegen_emit_stub_compiler **************************************************
2882
2883    Emit a stub routine which calls the compiler.
2884
2885 *******************************************************************************/
2886
2887 void codegen_emit_stub_compiler(jitdata *jd)
2888 {
2889         methodinfo  *m;
2890         codegendata *cd;
2891
2892         /* get required compiler data */
2893
2894         m  = jd->m;
2895         cd = jd->cd;
2896
2897         /* code for the stub */
2898
2899         M_LLD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));               /* methodinfo  */
2900 /*      M_ALD(REG_ITMP1, RIP, -(2 * SIZEOF_VOID_P));               /\* methodinfo  *\/ */
2901         M_LLD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));          /* compiler pointer */
2902 /*      M_ALD(REG_ITMP3, RIP, -(3 * SIZEOF_VOID_P));          /\* compiler pointer *\/ */
2903         M_JMP(REG_ITMP3);
2904 }
2905
2906
2907 /* codegen_emit_stub_builtin ***************************************************
2908
2909    Creates a stub routine which calls a builtin function.
2910
2911 *******************************************************************************/
2912
2913 void codegen_emit_stub_builtin(jitdata *jd, builtintable_entry *bte)
2914 {
2915         codeinfo    *code;
2916         codegendata *cd;
2917         methoddesc  *md;
2918         s4           i, j;
2919         s4           s1, disp;
2920
2921         /* get required compiler data */
2922
2923         code = jd->code;
2924         cd   = jd->cd;
2925
2926         md = bte->md;
2927
2928         /* calculate stack frame size */
2929
2930         cd->stackframesize =
2931                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
2932                 md->paramcount +                          /* saved argument registers */
2933                 1;                                                    /* return value */
2934
2935         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
2936
2937         /* create method header */
2938
2939         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2940         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2941         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
2942         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2943         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2944         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2945         (void) dseg_addlinenumbertablesize(cd);
2946         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
2947
2948         /* generate stub code */
2949
2950         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2951
2952 #if defined(ENABLE_GC_CACAO)
2953         /* Save callee saved integer registers in stackframeinfo (GC may
2954            need to recover them during a collection). */
2955
2956         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
2957                 OFFSET(stackframeinfo, intregs);
2958
2959         for (i = 0; i < INT_SAV_CNT; i++)
2960                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2961 #endif
2962
2963         /* save integer and float argument registers */
2964
2965         for (i = 0, j = 0; i < md->paramcount; i++) {
2966                 if (!md->params[i].inmemory) {
2967                         s1 = md->params[i].regoff;
2968
2969                         switch (md->paramtypes[i].type) {
2970                         case TYPE_INT:
2971                         case TYPE_LNG:
2972                         case TYPE_ADR:
2973                                 M_LST(s1, REG_SP, j * 8);
2974                                 break;
2975                         case TYPE_FLT:
2976                         case TYPE_DBL:
2977                                 M_DST(s1, REG_SP, j * 8);
2978                                 break;
2979                         }
2980
2981                         j++;
2982                 }
2983         }
2984
2985         /* create dynamic stack info */
2986
2987         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
2988         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
2989         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
2990         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
2991         M_MOV_IMM(codegen_stub_builtin_enter, REG_ITMP1);
2992         M_CALL(REG_ITMP1);
2993
2994         /* restore integer and float argument registers */
2995
2996         for (i = 0, j = 0; i < md->paramcount; i++) {
2997                 if (!md->params[i].inmemory) {
2998                         s1 = md->params[i].regoff;
2999
3000                         switch (md->paramtypes[i].type) {
3001                         case TYPE_INT:
3002                         case TYPE_LNG:
3003                         case TYPE_ADR:
3004                                 M_LLD(s1, REG_SP, j * 8);
3005                                 break;
3006                         case TYPE_FLT:
3007                         case TYPE_DBL:
3008                                 M_DLD(s1, REG_SP, j * 8);
3009                                 break;
3010                         }
3011
3012                         j++;
3013                 }
3014         }
3015
3016         /* call the builtin function */
3017
3018         M_MOV_IMM(bte->fp, REG_ITMP3);
3019         M_CALL(REG_ITMP3);
3020
3021         /* save return value */
3022
3023         switch (md->returntype.type) {
3024         case TYPE_INT:
3025         case TYPE_LNG:
3026         case TYPE_ADR:
3027                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3028                 break;
3029         case TYPE_FLT:
3030         case TYPE_DBL:
3031                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3032                 break;
3033         case TYPE_VOID:
3034                 break;
3035         }
3036
3037         /* remove native stackframe info */
3038
3039         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3040         M_MOV_IMM(codegen_stub_builtin_exit, REG_ITMP1);
3041         M_CALL(REG_ITMP1);
3042
3043         /* restore return value */
3044
3045         switch (md->returntype.type) {
3046         case TYPE_INT:
3047         case TYPE_LNG:
3048         case TYPE_ADR:
3049                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3050                 break;
3051         case TYPE_FLT:
3052         case TYPE_DBL:
3053                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3054                 break;
3055         case TYPE_VOID:
3056                 break;
3057         }
3058
3059 #if defined(ENABLE_GC_CACAO)
3060         /* Restore callee saved integer registers from stackframeinfo (GC
3061            might have modified them during a collection). */
3062
3063         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3064                 OFFSET(stackframeinfo, intregs);
3065
3066         for (i = 0; i < INT_SAV_CNT; i++)
3067                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3068 #endif
3069
3070         /* remove stackframe */
3071
3072         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3073         M_RET;
3074 }
3075
3076
3077 /* codegen_emit_stub_native ****************************************************
3078
3079    Emits a stub routine which calls a native method.
3080
3081 *******************************************************************************/
3082
3083 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f)
3084 {
3085         methodinfo  *m;
3086         codeinfo    *code;
3087         codegendata *cd;
3088         methoddesc  *md;
3089         s4           nativeparams;
3090         s4           i, j;
3091         s4           t;
3092         s4           s1, s2;
3093         int          funcdisp;
3094 #if defined(ENABLE_GC_CACAO)
3095         int          disp;
3096 #endif
3097
3098         /* get required compiler data */
3099
3100         m    = jd->m;
3101         code = jd->code;
3102         cd   = jd->cd;
3103
3104         /* initialize variables */
3105
3106         md = m->parseddesc;
3107         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
3108
3109         /* calculate stack frame size */
3110
3111         cd->stackframesize =
3112                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
3113                 sizeof(localref_table) / SIZEOF_VOID_P +
3114                 md->paramcount +
3115                 1 +                       /* functionptr, TODO: store in data segment */
3116                 nmd->memuse;
3117
3118         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
3119
3120         /* create method header */
3121
3122         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
3123         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
3124         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
3125         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
3126         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
3127         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
3128         (void) dseg_addlinenumbertablesize(cd);
3129         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
3130
3131 #if defined(ENABLE_PROFILING)
3132         /* generate native method profiling code */
3133
3134         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3135                 /* count frequency */
3136
3137                 M_MOV_IMM(code, REG_ITMP3);
3138                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3139         }
3140 #endif
3141
3142         /* generate stub code */
3143
3144         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3145
3146         /* get function address (this must happen before the stackframeinfo) */
3147
3148         funcdisp = dseg_add_functionptr(cd, f);
3149
3150         if (f == NULL)
3151                 patcher_add_patch_ref(jd, PATCHER_resolve_native_function, m, funcdisp);
3152
3153 #if defined(ENABLE_GC_CACAO)
3154         /* Save callee saved integer registers in stackframeinfo (GC may
3155            need to recover them during a collection). */
3156
3157         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3158                 OFFSET(stackframeinfo, intregs);
3159
3160         for (i = 0; i < INT_SAV_CNT; i++)
3161                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3162 #endif
3163
3164         /* save integer and float argument registers */
3165
3166         for (i = 0; i < md->paramcount; i++) {
3167                 if (!md->params[i].inmemory) {
3168                         s1 = md->params[i].regoff;
3169
3170                         switch (md->paramtypes[i].type) {
3171                         case TYPE_INT:
3172                         case TYPE_LNG:
3173                         case TYPE_ADR:
3174                                 M_LST(s1, REG_SP, i * 8);
3175                                 break;
3176                         case TYPE_FLT:
3177                         case TYPE_DBL:
3178                                 M_DST(s1, REG_SP, i * 8);
3179                                 break;
3180                         }
3181                 }
3182         }
3183
3184         /* create dynamic stack info */
3185
3186         M_MOV(REG_SP, REG_A0);
3187         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3188         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3189         M_CALL(REG_ITMP1);
3190
3191         /* remember class argument */
3192
3193         if (m->flags & ACC_STATIC)
3194                 M_MOV(REG_RESULT, REG_ITMP2);
3195
3196         /* restore integer and float argument registers */
3197
3198         for (i = 0; i < md->paramcount; i++) {
3199                 if (!md->params[i].inmemory) {
3200                         s1 = md->params[i].regoff;
3201
3202                         switch (md->paramtypes[i].type) {
3203                         case TYPE_INT:
3204                         case TYPE_LNG:
3205                         case TYPE_ADR:
3206                                 M_LLD(s1, REG_SP, i * 8);
3207                                 break;
3208                         case TYPE_FLT:
3209                         case TYPE_DBL:
3210                                 M_DLD(s1, REG_SP, i * 8);
3211                                 break;
3212                         }
3213                 }
3214         }
3215
3216         /* copy or spill arguments to new locations */
3217
3218         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3219                 t  = md->paramtypes[i].type;
3220                 s2 = nmd->params[j].regoff;
3221
3222                 if (IS_INT_LNG_TYPE(t)) {
3223                         if (!md->params[i].inmemory) {
3224                                 s1 = md->params[i].regoff;
3225
3226                                 if (!nmd->params[j].inmemory)
3227                                         M_INTMOVE(s1, s2);
3228                                 else
3229                                         M_LST(s1, REG_SP, s2);
3230                         }
3231                         else {
3232                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3233                                 M_LLD(REG_ITMP1, REG_SP, s1);
3234                                 M_LST(REG_ITMP1, REG_SP, s2);
3235                         }
3236                 }
3237                 else {
3238                         /* We only copy spilled float arguments, as the float
3239                            argument registers keep unchanged. */
3240
3241                         if (md->params[i].inmemory) {
3242                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3243
3244                                 if (IS_2_WORD_TYPE(t)) {
3245                                         M_DLD(REG_FTMP1, REG_SP, s1);
3246                                         M_DST(REG_FTMP1, REG_SP, s2);
3247                                 }
3248                                 else {
3249                                         M_FLD(REG_FTMP1, REG_SP, s1);
3250                                         M_FST(REG_FTMP1, REG_SP, s2);
3251                                 }
3252                         }
3253                 }
3254         }
3255
3256         /* put class into second argument register */
3257
3258         if (m->flags & ACC_STATIC)
3259                 M_MOV(REG_ITMP2, REG_A1);
3260
3261         /* put env into first argument register */
3262
3263         M_MOV_IMM(_Jv_env, REG_A0);
3264
3265         /* do the native function call */
3266
3267         M_ALD(REG_ITMP1, RIP, funcdisp);
3268         M_CALL(REG_ITMP1);
3269
3270         /* save return value */
3271
3272         switch (md->returntype.type) {
3273         case TYPE_INT:
3274         case TYPE_LNG:
3275         case TYPE_ADR:
3276                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3277                 break;
3278         case TYPE_FLT:
3279         case TYPE_DBL:
3280                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3281                 break;
3282         case TYPE_VOID:
3283                 break;
3284         }
3285
3286         /* remove native stackframe info */
3287
3288         M_MOV(REG_SP, REG_A0);
3289         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3290         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3291         M_CALL(REG_ITMP1);
3292         M_MOV(REG_RESULT, REG_ITMP3);
3293
3294         /* restore return value */
3295
3296         switch (md->returntype.type) {
3297         case TYPE_INT:
3298         case TYPE_LNG:
3299         case TYPE_ADR:
3300                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3301                 break;
3302         case TYPE_FLT:
3303         case TYPE_DBL:
3304                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3305                 break;
3306         case TYPE_VOID:
3307                 break;
3308         }
3309
3310 #if defined(ENABLE_GC_CACAO)
3311         /* Restore callee saved integer registers from stackframeinfo (GC
3312            might have modified them during a collection). */
3313
3314         disp = cd->stackframesize * 8 - sizeof(stackframeinfo) +
3315                 OFFSET(stackframeinfo, intregs);
3316
3317         for (i = 0; i < INT_SAV_CNT; i++)
3318                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3319 #endif
3320
3321         /* remove stackframe */
3322
3323         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3324
3325         /* test for exception */
3326
3327         M_TEST(REG_ITMP3);
3328         M_BNE(1);
3329         M_RET;
3330
3331         /* handle exception */
3332
3333         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3334         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3335         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3336
3337         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3338         M_JMP(REG_ITMP3);
3339
3340         /* Generate patcher traps. */
3341
3342         emit_patcher_traps(jd);
3343 }
3344
3345
3346 /*
3347  * These are local overrides for various environment variables in Emacs.
3348  * Please do not remove this and leave it at the end of the file, where
3349  * Emacs will automagically detect them.
3350  * ---------------------------------------------------------------------
3351  * Local variables:
3352  * mode: c
3353  * indent-tabs-mode: t
3354  * c-basic-offset: 4
3355  * tab-width: 4
3356  * End:
3357  * vim:noexpandtab:sw=4:ts=4:
3358  */