src/vm/jit/i386/codegen.c

   1 /* src/vm/jit/i386/codegen.c - machine code generator for i386
   2
   3    Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25 */
  26
  27
  28 #include "config.h"
  29
  30 #include <assert.h>
  31 #include <stdint.h>
  32 #include <stdio.h>
  33
  34 #include "vm/types.h"
  35
  36 #include "vm/jit/i386/md-abi.h"
  37
  38 #include "vm/jit/i386/codegen.h"
  39 #include "vm/jit/i386/emit.h"
  40
  41 #include "mm/memory.h"
  42 #include "native/jni.h"
  43 #include "native/localref.h"
  44 #include "native/native.h"
  45
  46 #include "threads/lock-common.h"
  47
  48 #include "vm/builtin.h"
  49 #include "vm/exceptions.h"
  50 #include "vm/global.h"
  51 #include "vm/stringlocal.h"
  52 #include "vm/vm.h"
  53
  54 #include "vm/jit/abi.h"
  55 #include "vm/jit/asmpart.h"
  56 #include "vm/jit/codegen-common.h"
  57 #include "vm/jit/dseg.h"
  58 #include "vm/jit/emit-common.h"
  59 #include "vm/jit/jit.h"
  60 #include "vm/jit/linenumbertable.h"
  61 #include "vm/jit/parse.h"
  62 #include "vm/jit/patcher-common.h"
  63 #include "vm/jit/reg.h"
  64 #include "vm/jit/replace.h"
  65 #include "vm/jit/stacktrace.h"
  66
  67 #if defined(ENABLE_SSA)
  68 # include "vm/jit/optimizing/lsra.h"
  69 # include "vm/jit/optimizing/ssa.h"
  70 #elif defined(ENABLE_LSRA)
  71 # include "vm/jit/allocator/lsra.h"
  72 #endif
  73
  74 #include "vmcore/loader.h"
  75 #include "vmcore/options.h"
  76 #include "vmcore/utf8.h"
  77
  78
  79 /* codegen_emit ****************************************************************
  80
  81    Generates machine code.
  82
  83 *******************************************************************************/
  84
  85 bool codegen_emit(jitdata *jd)
  86 {
  87         methodinfo         *m;
  88         codeinfo           *code;
  89         codegendata        *cd;
  90         registerdata       *rd;
  91         s4                  len, s1, s2, s3, d, disp;
  92         int                 align_off;      /* offset for alignment compensation  */
  93         varinfo            *var, *var1;
  94         basicblock         *bptr;
  95         instruction        *iptr;
  96         u2                  currentline;
  97         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
  98         builtintable_entry *bte;
  99         methoddesc         *md;
 100         fieldinfo          *fi;
 101         unresolved_field   *uf;
 102         s4                  fieldtype;
 103         s4                  varindex;
 104 #if defined(ENABLE_SSA)
 105         lsradata *ls;
 106         bool last_cmd_was_goto;
 107
 108         last_cmd_was_goto = false;
 109         ls = jd->ls;
 110 #endif
 111
 112         /* get required compiler data */
 113
 114         m    = jd->m;
 115         code = jd->code;
 116         cd   = jd->cd;
 117         rd   = jd->rd;
 118
 119         /* prevent compiler warnings */
 120
 121         s1          = 0;
 122         s2          = 0;
 123         d           = 0;
 124         currentline = 0;
 125         lm          = NULL;
 126         bte         = NULL;
 127
 128         {
 129         s4 i, p, t, l;
 130         s4 savedregs_num = 0;
 131         s4 stack_off = 0;
 132
 133         /* space to save used callee saved registers */
 134
 135         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 136         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 137
 138         cd->stackframesize = rd->memuse + savedregs_num;
 139
 140
 141 #if defined(ENABLE_THREADS)
 142         /* space to save argument of monitor_enter */
 143
 144         if (checksync && code_is_synchronized(code))
 145                 cd->stackframesize++;
 146 #endif
 147
 148         /* create method header */
 149
 150     /* Keep stack of non-leaf functions 16-byte aligned. */
 151
 152         if (!code_is_leafmethod(code)) {
 153                 ALIGN_ODD(cd->stackframesize);
 154         }
 155
 156         align_off = cd->stackframesize ? 4 : 0;
 157
 158         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 159         (void) dseg_add_unique_s4(
 160                 cd, cd->stackframesize * 8 + align_off);           /* FrameSize       */
 161
 162         code->synchronizedoffset = rd->memuse * 8;
 163
 164         /* REMOVEME: We still need it for exception handling in assembler. */
 165
 166         if (code_is_leafmethod(code))
 167                 (void) dseg_add_unique_s4(cd, 1);                  /* IsLeaf          */
 168         else
 169                 (void) dseg_add_unique_s4(cd, 0);                  /* IsLeaf          */
 170
 171         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 172         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 173
 174 #if defined(ENABLE_PROFILING)
 175         /* generate method profiling code */
 176
 177         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 178                 /* count frequency */
 179
 180                 M_MOV_IMM(code, REG_ITMP3);
 181                 M_IADD_IMM_MEMBASE(1, REG_ITMP3, OFFSET(codeinfo, frequency));
 182         }
 183 #endif
 184
 185         /* create stack frame (if necessary) */
 186
 187         if (cd->stackframesize)
 188                 /* align_off == 4 */
 189                 M_ASUB_IMM(cd->stackframesize * 8 + 4, REG_SP);
 190
 191         /* save return address and used callee saved registers */
 192
 193         p = cd->stackframesize;
 194         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 195                 p--; M_AST(rd->savintregs[i], REG_SP, p * 8);
 196         }
 197         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 198                 p--; emit_fld_reg(cd, rd->savfltregs[i]); emit_fstpl_membase(cd, REG_SP, p * 8);
 199         }
 200
 201         /* take arguments out of register or stack frame */
 202
 203         md = m->parseddesc;
 204
 205         stack_off = 0;
 206         for (p = 0, l = 0; p < md->paramcount; p++) {
 207                 t = md->paramtypes[p].type;
 208
 209                 varindex = jd->local_map[l * 5 + t];
 210 #if defined(ENABLE_SSA)
 211                 if ( ls != NULL ) {
 212                         if (varindex != UNUSED)
 213                                 varindex = ls->var_0[varindex];
 214                         if ((varindex != UNUSED) && (ls->lifetime[varindex].type == UNUSED))
 215                                 varindex = UNUSED;
 216                 }
 217 #endif
 218                 l++;
 219                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 220                         l++;
 221
 222                 if (varindex == UNUSED)
 223                         continue;
 224
 225                 var = VAR(varindex);
 226                 s1  = md->params[p].regoff;
 227                 d   = var->vv.regoff;
 228
 229                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 230                         if (!md->params[p].inmemory) {           /* register arguments    */
 231                                 log_text("integer register argument");
 232                                 assert(0);
 233                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 234                                         /* rd->argintregs[md->params[p].regoff -> var->vv.regoff     */
 235                                 }
 236                                 else {                               /* reg arg -> spilled    */
 237                                         /* rd->argintregs[md->params[p].regoff -> var->vv.regoff * 4 */
 238                                 }
 239                         }
 240                         else {
 241                                 if (!(var->flags & INMEMORY)) {
 242                                         M_ILD(d, REG_SP,
 243                                                   cd->stackframesize * 8 + 4 + align_off + s1);
 244                                 }
 245                                 else {
 246                                         if (!IS_2_WORD_TYPE(t)) {
 247 #if defined(ENABLE_SSA)
 248                                                 /* no copy avoiding by now possible with SSA */
 249                                                 if (ls != NULL) {
 250                                                         emit_mov_membase_reg(   /* + 4 for return address */
 251                                                                 cd, REG_SP,
 252                                                                 cd->stackframesize * 8 + s1 + 4 + align_off,
 253                                                                 REG_ITMP1);
 254                                                         emit_mov_reg_membase(
 255                                                                 cd, REG_ITMP1, REG_SP, var->vv.regoff);
 256                                                 }
 257                                                 else
 258 #endif /*defined(ENABLE_SSA)*/
 259                                                         /* reuse stackslot */
 260                                                         var->vv.regoff = cd->stackframesize * 8 + 4 +
 261                                                                 align_off + s1;
 262
 263                                         }
 264                                         else {
 265 #if defined(ENABLE_SSA)
 266                                                 /* no copy avoiding by now possible with SSA */
 267                                                 if (ls != NULL) {
 268                                                         emit_mov_membase_reg(  /* + 4 for return address */
 269                                                                 cd, REG_SP,
 270                                                                 cd->stackframesize * 8 + s1 + 4 + align_off,
 271                                                                 REG_ITMP1);
 272                                                         emit_mov_reg_membase(
 273                                                                 cd, REG_ITMP1, REG_SP, var->vv.regoff);
 274                                                         emit_mov_membase_reg(   /* + 4 for return address */
 275                                                                 cd, REG_SP,
 276                                                                 cd->stackframesize * 8 + s1 + 4 + 4 + align_off,
 277                                                                 REG_ITMP1);
 278                                                         emit_mov_reg_membase(
 279                                                                 cd, REG_ITMP1, REG_SP, var->vv.regoff + 4);
 280                                                 }
 281                                                 else
 282 #endif /*defined(ENABLE_SSA)*/
 283                                                         /* reuse stackslot */
 284                                                         var->vv.regoff = cd->stackframesize * 8 + 8 + s1;
 285                                         }
 286                                 }
 287                         }
 288                 }
 289                 else {                                       /* floating args         */
 290                         if (!md->params[p].inmemory) {           /* register arguments    */
 291                                 log_text("There are no float argument registers!");
 292                                 assert(0);
 293                                 if (!(var->flags & INMEMORY)) {  /* reg arg -> register   */
 294                                         /* rd->argfltregs[md->params[p].regoff -> var->vv.regoff     */
 295                                 } else {                                     /* reg arg -> spilled    */
 296                                         /* rd->argfltregs[md->params[p].regoff -> var->vv.regoff * 8 */
 297                                 }
 298
 299                         }
 300                         else {                                   /* stack arguments       */
 301                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 302                                         if (t == TYPE_FLT) {
 303                                                 emit_flds_membase(
 304                             cd, REG_SP,
 305                                                         cd->stackframesize * 8 + s1 + 4 + align_off);
 306                                                 assert(0);
 307 /*                                              emit_fstp_reg(cd, var->vv.regoff + fpu_st_offset); */
 308
 309                                         }
 310                                         else {
 311                                                 emit_fldl_membase(
 312                             cd, REG_SP,
 313                                                         cd->stackframesize * 8 + s1 + 4 + align_off);
 314                                                 assert(0);
 315 /*                                              emit_fstp_reg(cd, var->vv.regoff + fpu_st_offset); */
 316                                         }
 317
 318                                 } else {                             /* stack-arg -> spilled  */
 319 #if defined(ENABLE_SSA)
 320                                         /* no copy avoiding by now possible with SSA */
 321                                         if (ls != NULL) {
 322                                                 emit_mov_membase_reg(
 323                                                         cd, REG_SP,
 324                                                         cd->stackframesize * 8 + s1 + 4 + align_off,
 325                                                         REG_ITMP1);
 326                                                 emit_mov_reg_membase(
 327                                                         cd, REG_ITMP1, REG_SP, var->vv.regoff);
 328                                                 if (t == TYPE_FLT) {
 329                                                         emit_flds_membase(
 330                                                                 cd, REG_SP,
 331                                                                 cd->stackframesize * 8 + s1 + 4 + align_off);
 332                                                         emit_fstps_membase(cd, REG_SP, var->vv.regoff);
 333                                                 }
 334                                                 else {
 335                                                         emit_fldl_membase(
 336                                                                 cd, REG_SP,
 337                                                                 cd->stackframesize * 8 + s1 + 4 + align_off);
 338                                                         emit_fstpl_membase(cd, REG_SP, var->vv.regoff);
 339                                                 }
 340                                         }
 341                                         else
 342 #endif /*defined(ENABLE_SSA)*/
 343                                                 /* reuse stackslot */
 344                                                 var->vv.regoff = cd->stackframesize * 8 + 4 +
 345                                                         align_off + s1;
 346                                 }
 347                         }
 348                 }
 349         }
 350
 351         /* call monitorenter function */
 352
 353 #if defined(ENABLE_THREADS)
 354         if (checksync && code_is_synchronized(code)) {
 355                 s1 = rd->memuse;
 356
 357                 if (m->flags & ACC_STATIC) {
 358                         M_MOV_IMM(&m->class->object.header, REG_ITMP1);
 359                 }
 360                 else {
 361                         M_ALD(REG_ITMP1, REG_SP, cd->stackframesize * 8 + 4 + align_off);
 362                         M_TEST(REG_ITMP1);
 363                         M_BNE(6);
 364                         M_ALD_MEM(REG_ITMP1, EXCEPTION_HARDWARE_NULLPOINTER);
 365                 }
 366
 367                 M_AST(REG_ITMP1, REG_SP, s1 * 8);
 368                 M_AST(REG_ITMP1, REG_SP, 0 * 4);
 369                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP3);
 370                 M_CALL(REG_ITMP3);
 371         }
 372 #endif
 373
 374 #if !defined(NDEBUG)
 375         emit_verbosecall_enter(jd);
 376 #endif
 377
 378         }
 379
 380 #if defined(ENABLE_SSA)
 381         /* with SSA the Header is Basic Block 0 - insert phi Moves if necessary */
 382         if ( ls != NULL)
 383                 codegen_emit_phi_moves(jd, ls->basicblocks[0]);
 384 #endif
 385
 386         /* end of header generation */
 387
 388         /* create replacement points */
 389
 390         REPLACEMENT_POINTS_INIT(cd, jd);
 391
 392         /* walk through all basic blocks */
 393
 394         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 395
 396                 bptr->mpc = (s4) (cd->mcodeptr - cd->mcodebase);
 397
 398                 if (bptr->flags >= BBREACHED) {
 399                 /* branch resolving */
 400
 401                 codegen_resolve_branchrefs(cd, bptr);
 402
 403                 /* handle replacement points */
 404
 405                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 406
 407 #if defined(ENABLE_REPLACEMENT)
 408                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 409                         if (cd->replacementpoint[-1].flags & RPLPOINT_FLAG_COUNTDOWN) {
 410                                 MCODECHECK(32);
 411                                 disp = (s4) &(m->hitcountdown);
 412                                 M_ISUB_IMM_MEMABS(1, disp);
 413                                 M_BS(0);
 414                         }
 415                 }
 416 #endif
 417
 418                 /* copy interface registers to their destination */
 419
 420                 len = bptr->indepth;
 421                 MCODECHECK(512);
 422
 423 #if defined(ENABLE_PROFILING)
 424                 /* generate basic block profiling code */
 425
 426                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 427                         /* count frequency */
 428
 429                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 430                         M_IADD_IMM_MEMBASE(1, REG_ITMP3, bptr->nr * 4);
 431                 }
 432 #endif
 433
 434 #if defined(ENABLE_LSRA) || defined(ENABLE_SSA)
 435 # if defined(ENABLE_LSRA) && !defined(ENABLE_SSA)
 436                 if (opt_lsra) {
 437 # endif
 438 # if defined(ENABLE_SSA)
 439                 if (ls != NULL) {
 440                         last_cmd_was_goto = false;
 441 # endif
 442                         if (len > 0) {
 443                                 len--;
 444                                 var = VAR(bptr->invars[len]);
 445                                 if (bptr->type != BBTYPE_STD) {
 446                                         if (!IS_2_WORD_TYPE(var->type)) {
 447                                                 if (bptr->type == BBTYPE_EXH) {
 448                                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 449                                                         M_INTMOVE(REG_ITMP1, d);
 450                                                         emit_store(jd, NULL, var, d);
 451                                                 }
 452                                         }
 453                                         else {
 454                                                 log_text("copy interface registers(EXH, SBR): longs \
 455                                   have to be in memory (begin 1)");
 456                                                 assert(0);
 457                                         }
 458                                 }
 459                         }
 460
 461                 }
 462                 else
 463 #endif /* defined(ENABLE_LSRA) || defined(ENABLE_SSA) */
 464                 {
 465                 while (len) {
 466                         len--;
 467                         var = VAR(bptr->invars[len]);
 468                         if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 469                                 if (!IS_2_WORD_TYPE(var->type)) {
 470                                         if (bptr->type == BBTYPE_EXH) {
 471                                                 d = codegen_reg_of_var(0, var, REG_ITMP1);
 472                                                 M_INTMOVE(REG_ITMP1, d);
 473                                                 emit_store(jd, NULL, var, d);
 474                                         }
 475                                 }
 476                                 else {
 477                                         log_text("copy interface registers: longs have to be in \
 478                                memory (begin 1)");
 479                                         assert(0);
 480                                 }
 481
 482                         }
 483                         else {
 484                                 assert((var->flags & INOUT));
 485                         }
 486                 } /* while (len) */
 487                 } /* */
 488
 489                 /* walk through all instructions */
 490
 491                 len = bptr->icount;
 492                 currentline = 0;
 493
 494                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 495                         if (iptr->line != currentline) {
 496                                 linenumbertable_list_entry_add(cd, iptr->line);
 497                                 currentline = iptr->line;
 498                         }
 499
 500                         MCODECHECK(1024);                         /* 1kB should be enough */
 501
 502                 switch (iptr->opc) {
 503                 case ICMD_NOP:        /* ...  ==> ...                                 */
 504                 case ICMD_POP:        /* ..., value  ==> ...                          */
 505                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 506                         break;
 507
 508                 case ICMD_INLINE_START:
 509
 510                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 511                         break;
 512
 513                 case ICMD_INLINE_BODY:
 514
 515                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 516                         linenumbertable_list_entry_add_inline_start(cd, iptr);
 517                         linenumbertable_list_entry_add(cd, iptr->line);
 518                         break;
 519
 520                 case ICMD_INLINE_END:
 521
 522                         linenumbertable_list_entry_add_inline_end(cd, iptr);
 523                         linenumbertable_list_entry_add(cd, iptr->line);
 524                         break;
 525
 526                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 527
 528                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 529                         emit_nullpointer_check(cd, iptr, s1);
 530                         break;
 531
 532                 /* constant operations ************************************************/
 533
 534                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 535
 536                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 537                         ICONST(d, iptr->sx.val.i);
 538                         emit_store_dst(jd, iptr, d);
 539                         break;
 540
 541                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 542
 543                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP12_PACKED);
 544                         LCONST(d, iptr->sx.val.l);
 545                         emit_store_dst(jd, iptr, d);
 546                         break;
 547
 548                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 549
 550                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 551                         if (iptr->sx.val.f == 0.0) {
 552                                 emit_fldz(cd);
 553
 554                                 /* -0.0 */
 555                                 if (iptr->sx.val.i == 0x80000000) {
 556                                         emit_fchs(cd);
 557                                 }
 558
 559                         } else if (iptr->sx.val.f == 1.0) {
 560                                 emit_fld1(cd);
 561
 562                         } else if (iptr->sx.val.f == 2.0) {
 563                                 emit_fld1(cd);
 564                                 emit_fld1(cd);
 565                                 emit_faddp(cd);
 566
 567                         } else {
 568                                 disp = dseg_add_float(cd, iptr->sx.val.f);
 569                                 emit_mov_imm_reg(cd, 0, REG_ITMP1);
 570                                 dseg_adddata(cd);
 571                                 emit_flds_membase(cd, REG_ITMP1, disp);
 572                         }
 573                         emit_store_dst(jd, iptr, d);
 574                         break;
 575
 576                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 577
 578                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 579                         if (iptr->sx.val.d == 0.0) {
 580                                 emit_fldz(cd);
 581
 582                                 /* -0.0 */
 583                                 if (iptr->sx.val.l == 0x8000000000000000LL) {
 584                                         emit_fchs(cd);
 585                                 }
 586
 587                         } else if (iptr->sx.val.d == 1.0) {
 588                                 emit_fld1(cd);
 589
 590                         } else if (iptr->sx.val.d == 2.0) {
 591                                 emit_fld1(cd);
 592                                 emit_fld1(cd);
 593                                 emit_faddp(cd);
 594
 595                         } else {
 596                                 disp = dseg_add_double(cd, iptr->sx.val.d);
 597                                 emit_mov_imm_reg(cd, 0, REG_ITMP1);
 598                                 dseg_adddata(cd);
 599                                 emit_fldl_membase(cd, REG_ITMP1, disp);
 600                         }
 601                         emit_store_dst(jd, iptr, d);
 602                         break;
 603
 604                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 605
 606                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 607
 608                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 609                                 patcher_add_patch_ref(jd, PATCHER_aconst,
 610                                                                         iptr->sx.val.c.ref, 0);
 611
 612                                 M_MOV_IMM(NULL, d);
 613
 614                         } else {
 615                                 if (iptr->sx.val.anyptr == NULL)
 616                                         M_CLR(d);
 617                                 else
 618                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 619                         }
 620                         emit_store_dst(jd, iptr, d);
 621                         break;
 622
 623
 624                 /* load/store/copy/move operations ************************************/
 625
 626                 case ICMD_ILOAD:
 627                 case ICMD_ALOAD:
 628                 case ICMD_LLOAD:
 629                 case ICMD_FLOAD:
 630                 case ICMD_DLOAD:
 631                 case ICMD_ISTORE:
 632                 case ICMD_LSTORE:
 633                 case ICMD_FSTORE:
 634                 case ICMD_DSTORE:
 635                 case ICMD_COPY:
 636                 case ICMD_MOVE:
 637
 638                         emit_copy(jd, iptr);
 639                         break;
 640
 641                 case ICMD_ASTORE:
 642                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 643                                 emit_copy(jd, iptr);
 644                         break;
 645
 646
 647                 /* integer operations *************************************************/
 648
 649                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 650
 651                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 652                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 653                         M_INTMOVE(s1, d);
 654                         M_NEG(d);
 655                         emit_store_dst(jd, iptr, d);
 656                         break;
 657
 658                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 659
 660                         s1 = emit_load_s1(jd, iptr, REG_ITMP12_PACKED);
 661                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP12_PACKED);
 662                         M_LNGMOVE(s1, d);
 663                         M_NEG(GET_LOW_REG(d));
 664                         M_IADDC_IMM(0, GET_HIGH_REG(d));
 665                         M_NEG(GET_HIGH_REG(d));
 666                         emit_store_dst(jd, iptr, d);
 667                         break;
 668
 669                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 670
 671                         s1 = emit_load_s1(jd, iptr, EAX);
 672                         d = codegen_reg_of_dst(jd, iptr, EAX_EDX_PACKED);
 673                         M_INTMOVE(s1, EAX);
 674                         M_CLTD;
 675                         M_LNGMOVE(EAX_EDX_PACKED, d);
 676                         emit_store_dst(jd, iptr, d);
 677                         break;
 678
 679                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 680
 681                         s1 = emit_load_s1_low(jd, iptr, REG_ITMP2);
 682                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 683                         M_INTMOVE(s1, d);
 684                         emit_store_dst(jd, iptr, d);
 685                         break;
 686
 687                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 688
 689                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 690                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 691                         M_INTMOVE(s1, d);
 692                         M_SLL_IMM(24, d);
 693                         M_SRA_IMM(24, d);
 694                         emit_store_dst(jd, iptr, d);
 695                         break;
 696
 697                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 698
 699                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 700                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 701                         M_CZEXT(s1, d);
 702                         emit_store_dst(jd, iptr, d);
 703                         break;
 704
 705                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 706
 707                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 708                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 709                         M_SSEXT(s1, d);
 710                         emit_store_dst(jd, iptr, d);
 711                         break;
 712
 713
 714                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 715
 716                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 717                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 718                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 719                         if (s2 == d)
 720                                 M_IADD(s1, d);
 721                         else {
 722                                 M_INTMOVE(s1, d);
 723                                 M_IADD(s2, d);
 724                         }
 725                         emit_store_dst(jd, iptr, d);
 726                         break;
 727
 728                 case ICMD_IINC:
 729                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 730                                       /* sx.val.i = constant                          */
 731
 732                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 733                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 734
 735                         /* `inc reg' is slower on p4's (regarding to ia32
 736                            optimization reference manual and benchmarks) and as
 737                            fast on athlon's. */
 738
 739                         M_INTMOVE(s1, d);
 740                         M_IADD_IMM(iptr->sx.val.i, d);
 741                         emit_store_dst(jd, iptr, d);
 742                         break;
 743
 744                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 745
 746                         s1 = emit_load_s1_low(jd, iptr, REG_ITMP1);
 747                         s2 = emit_load_s2_low(jd, iptr, REG_ITMP2);
 748                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP12_PACKED);
 749                         M_INTMOVE(s1, GET_LOW_REG(d));
 750                         M_IADD(s2, GET_LOW_REG(d));
 751                         /* don't use REG_ITMP1 */
 752                         s1 = emit_load_s1_high(jd, iptr, REG_ITMP2);
 753                         s2 = emit_load_s2_high(jd, iptr, REG_ITMP3);
 754                         M_INTMOVE(s1, GET_HIGH_REG(d));
 755                         M_IADDC(s2, GET_HIGH_REG(d));
 756                         emit_store_dst(jd, iptr, d);
 757                         break;
 758
 759                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 760                                       /* sx.val.l = constant                          */
 761
 762                         s1 = emit_load_s1(jd, iptr, REG_ITMP12_PACKED);
 763                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP12_PACKED);
 764                         M_LNGMOVE(s1, d);
 765                         M_IADD_IMM(iptr->sx.val.l, GET_LOW_REG(d));
 766                         M_IADDC_IMM(iptr->sx.val.l >> 32, GET_HIGH_REG(d));
 767                         emit_store_dst(jd, iptr, d);
 768                         break;
 769
 770                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 771
 772                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 773                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 774                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 775                         if (s2 == d) {
 776                                 M_INTMOVE(s1, REG_ITMP1);
 777                                 M_ISUB(s2, REG_ITMP1);
 778                                 M_INTMOVE(REG_ITMP1, d);
 779                         }
 780                         else {
 781                                 M_INTMOVE(s1, d);
 782                                 M_ISUB(s2, d);
 783                         }
 784                         emit_store_dst(jd, iptr, d);
 785                         break;
 786
 787                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 788                                       /* sx.val.i = constant                             */
 789
 790                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 791                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 792                         M_INTMOVE(s1, d);
 793                         M_ISUB_IMM(iptr->sx.val.i, d);
 794                         emit_store_dst(jd, iptr, d);
 795                         break;
 796
 797                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 798
 799                         s1 = emit_load_s1_low(jd, iptr, REG_ITMP1);
 800                         s2 = emit_load_s2_low(jd, iptr, REG_ITMP2);
 801                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP12_PACKED);
 802                         if (s2 == GET_LOW_REG(d)) {
 803                                 M_INTMOVE(s1, REG_ITMP1);
 804                                 M_ISUB(s2, REG_ITMP1);
 805                                 M_INTMOVE(REG_ITMP1, GET_LOW_REG(d));
 806                         }
 807                         else {
 808                                 M_INTMOVE(s1, GET_LOW_REG(d));
 809                                 M_ISUB(s2, GET_LOW_REG(d));
 810                         }
 811                         /* don't use REG_ITMP1 */
 812                         s1 = emit_load_s1_high(jd, iptr, REG_ITMP2);
 813                         s2 = emit_load_s2_high(jd, iptr, REG_ITMP3);
 814                         if (s2 == GET_HIGH_REG(d)) {
 815                                 M_INTMOVE(s1, REG_ITMP2);
 816                                 M_ISUBB(s2, REG_ITMP2);
 817                                 M_INTMOVE(REG_ITMP2, GET_HIGH_REG(d));
 818                         }
 819                         else {
 820                                 M_INTMOVE(s1, GET_HIGH_REG(d));
 821                                 M_ISUBB(s2, GET_HIGH_REG(d));
 822                         }
 823                         emit_store_dst(jd, iptr, d);
 824                         break;
 825
 826                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 827                                       /* sx.val.l = constant                          */
 828
 829                         s1 = emit_load_s1(jd, iptr, REG_ITMP12_PACKED);
 830                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP12_PACKED);
 831                         M_LNGMOVE(s1, d);
 832                         M_ISUB_IMM(iptr->sx.val.l, GET_LOW_REG(d));
 833                         M_ISUBB_IMM(iptr->sx.val.l >> 32, GET_HIGH_REG(d));
 834                         emit_store_dst(jd, iptr, d);
 835                         break;
 836
 837                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 838
 839                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 840                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 841                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 842                         if (s2 == d)
 843                                 M_IMUL(s1, d);
 844                         else {
 845                                 M_INTMOVE(s1, d);
 846                                 M_IMUL(s2, d);
 847                         }
 848                         emit_store_dst(jd, iptr, d);
 849                         break;
 850
 851                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 852                                       /* sx.val.i = constant                          */
 853
 854                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 855                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 856                         M_IMUL_IMM(s1, iptr->sx.val.i, d);
 857                         emit_store_dst(jd, iptr, d);
 858                         break;
 859
 860                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 861
 862                         s1 = emit_load_s1_high(jd, iptr, REG_ITMP2);
 863                         s2 = emit_load_s2_low(jd, iptr, EDX);
 864                         d = codegen_reg_of_dst(jd, iptr, EAX_EDX_PACKED);
 865
 866                         M_INTMOVE(s1, REG_ITMP2);
 867                         M_IMUL(s2, REG_ITMP2);
 868
 869                         s1 = emit_load_s1_low(jd, iptr, EAX);
 870                         s2 = emit_load_s2_high(jd, iptr, EDX);
 871                         M_INTMOVE(s2, EDX);
 872                         M_IMUL(s1, EDX);
 873                         M_IADD(EDX, REG_ITMP2);
 874
 875                         s1 = emit_load_s1_low(jd, iptr, EAX);
 876                         s2 = emit_load_s2_low(jd, iptr, EDX);
 877                         M_INTMOVE(s1, EAX);
 878                         M_MUL(s2);
 879                         M_INTMOVE(EAX, GET_LOW_REG(d));
 880                         M_IADD(REG_ITMP2, GET_HIGH_REG(d));
 881
 882                         emit_store_dst(jd, iptr, d);
 883                         break;
 884
 885                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 886                                       /* sx.val.l = constant                          */
 887
 888                         s1 = emit_load_s1_low(jd, iptr, REG_ITMP2);
 889                         d = codegen_reg_of_dst(jd, iptr, EAX_EDX_PACKED);
 890                         ICONST(EAX, iptr->sx.val.l);
 891                         M_MUL(s1);
 892                         M_IMUL_IMM(s1, iptr->sx.val.l >> 32, REG_ITMP2);
 893                         M_IADD(REG_ITMP2, EDX);
 894                         s1 = emit_load_s1_high(jd, iptr, REG_ITMP2);
 895                         M_IMUL_IMM(s1, iptr->sx.val.l, REG_ITMP2);
 896                         M_IADD(REG_ITMP2, EDX);
 897                         M_LNGMOVE(EAX_EDX_PACKED, d);
 898                         emit_store_dst(jd, iptr, d);
 899                         break;
 900
 901                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 902
 903                         s1 = emit_load_s1(jd, iptr, EAX);
 904                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 905                         d = codegen_reg_of_dst(jd, iptr, EAX);
 906                         emit_arithmetic_check(cd, iptr, s2);
 907
 908                         M_INTMOVE(s1, EAX);           /* we need the first operand in EAX */
 909
 910                         /* check as described in jvm spec */
 911
 912                         M_CMP_IMM(0x80000000, EAX);
 913                         M_BNE(3 + 6);
 914                         M_CMP_IMM(-1, s2);
 915                         M_BEQ(1 + 2);
 916                         M_CLTD;
 917                         M_IDIV(s2);
 918
 919                         M_INTMOVE(EAX, d);           /* if INMEMORY then d is already EAX */
 920                         emit_store_dst(jd, iptr, d);
 921                         break;
 922
 923                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 924
 925                         s1 = emit_load_s1(jd, iptr, EAX);
 926                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 927                         d = codegen_reg_of_dst(jd, iptr, EDX);
 928                         emit_arithmetic_check(cd, iptr, s2);
 929
 930                         M_INTMOVE(s1, EAX);           /* we need the first operand in EAX */
 931
 932                         /* check as described in jvm spec */
 933
 934                         M_CMP_IMM(0x80000000, EAX);
 935                         M_BNE(2 + 3 + 6);
 936                         M_CLR(EDX);
 937                         M_CMP_IMM(-1, s2);
 938                         M_BEQ(1 + 2);
 939                         M_CLTD;
 940                         M_IDIV(s2);
 941
 942                         M_INTMOVE(EDX, d);           /* if INMEMORY then d is already EDX */
 943                         emit_store_dst(jd, iptr, d);
 944                         break;
 945
 946                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 947                                       /* sx.val.i = constant                          */
 948
 949                         /* TODO: optimize for `/ 2' */
 950                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 951                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 952                         M_INTMOVE(s1, d);
 953                         M_TEST(d);
 954                         M_BNS(6);
 955                         M_IADD_IMM32((1 << iptr->sx.val.i) - 1, d);/* 32-bit for jump off */
 956                         M_SRA_IMM(iptr->sx.val.i, d);
 957                         emit_store_dst(jd, iptr, d);
 958                         break;
 959
 960                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 961                                       /* sx.val.i = constant                          */
 962
 963                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 964                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 965                         if (s1 == d) {
 966                                 M_MOV(s1, REG_ITMP1);
 967                                 s1 = REG_ITMP1;
 968                         }
 969                         M_INTMOVE(s1, d);
 970                         M_AND_IMM(iptr->sx.val.i, d);
 971                         M_TEST(s1);
 972                         M_BGE(2 + 2 + 6 + 2);
 973                         M_MOV(s1, d);  /* don't use M_INTMOVE, so we know the jump offset */
 974                         M_NEG(d);
 975                         M_AND_IMM32(iptr->sx.val.i, d);     /* use 32-bit for jump offset */
 976                         M_NEG(d);
 977                         emit_store_dst(jd, iptr, d);
 978                         break;
 979
 980                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 981                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 982
 983                         s2 = emit_load_s2(jd, iptr, REG_ITMP12_PACKED);
 984                         d = codegen_reg_of_dst(jd, iptr, REG_RESULT_PACKED);
 985
 986                         M_INTMOVE(GET_LOW_REG(s2), REG_ITMP3);
 987                         M_OR(GET_HIGH_REG(s2), REG_ITMP3);
 988                         /* XXX could be optimized */
 989                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 990
 991                         bte = iptr->sx.s23.s3.bte;
 992                         md = bte->md;
 993
 994                         M_LST(s2, REG_SP, 2 * 4);
 995
 996                         s1 = emit_load_s1(jd, iptr, REG_ITMP12_PACKED);
 997                         M_LST(s1, REG_SP, 0 * 4);
 998
 999                         M_MOV_IMM(bte->fp, REG_ITMP3);
1000                         M_CALL(REG_ITMP3);
1001                         emit_store_dst(jd, iptr, d);
1002                         break;
1003
1004                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1005                                       /* sx.val.i = constant                          */
1006
1007                         s1 = emit_load_s1(jd, iptr, REG_ITMP12_PACKED);
1008                         d = codegen_reg_of_dst(jd, iptr, REG_RESULT_PACKED);
1009                         M_LNGMOVE(s1, d);
1010                         M_TEST(GET_HIGH_REG(d));
1011                         M_BNS(6 + 3);
1012                         M_IADD_IMM32((1 << iptr->sx.val.i) - 1, GET_LOW_REG(d));
1013                         M_IADDC_IMM(0, GET_HIGH_REG(d));
1014                         M_SRLD_IMM(iptr->sx.val.i, GET_HIGH_REG(d), GET_LOW_REG(d));
1015                         M_SRA_IMM(iptr->sx.val.i, GET_HIGH_REG(d));
1016                         emit_store_dst(jd, iptr, d);
1017                         break;
1018
1019 #if 0
1020                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1021                                       /* sx.val.l = constant                          */
1022
1023                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1024                         if (iptr->dst.var->flags & INMEMORY) {
1025                                 if (iptr->s1.var->flags & INMEMORY) {
1026                                         /* Alpha algorithm */
1027                                         disp = 3;
1028                                         CALCOFFSETBYTES(disp, REG_SP, iptr->s1.var->vv.regoff * 8);
1029                                         disp += 3;
1030                                         CALCOFFSETBYTES(disp, REG_SP, iptr->s1.var->vv.regoff * 8 + 4);
1031
1032                                         disp += 2;
1033                                         disp += 3;
1034                                         disp += 2;
1035
1036                                         /* TODO: hmm, don't know if this is always correct */
1037                                         disp += 2;
1038                                         CALCIMMEDIATEBYTES(disp, iptr->sx.val.l & 0x00000000ffffffff);
1039                                         disp += 2;
1040                                         CALCIMMEDIATEBYTES(disp, iptr->sx.val.l >> 32);
1041
1042                                         disp += 2;
1043                                         disp += 3;
1044                                         disp += 2;
1045
1046                                         emit_mov_membase_reg(cd, REG_SP, iptr->s1.var->vv.regoff * 8, REG_ITMP1);
1047                                         emit_mov_membase_reg(cd, REG_SP, iptr->s1.var->vv.regoff * 8 + 4, REG_ITMP2);
1048
1049                                         emit_alu_imm_reg(cd, ALU_AND, iptr->sx.val.l, REG_ITMP1);
1050                                         emit_alu_imm_reg(cd, ALU_AND, iptr->sx.val.l >> 32, REG_ITMP2);
1051                                         emit_alu_imm_membase(cd, ALU_CMP, 0, REG_SP, iptr->s1.var->vv.regoff * 8 + 4);
1052                                         emit_jcc(cd, CC_GE, disp);
1053
1054                                         emit_mov_membase_reg(cd, REG_SP, iptr->s1.var->vv.regoff * 8, REG_ITMP1);
1055                                         emit_mov_membase_reg(cd, REG_SP, iptr->s1.var->vv.regoff * 8 + 4, REG_ITMP2);
1056
1057                                         emit_neg_reg(cd, REG_ITMP1);
1058                                         emit_alu_imm_reg(cd, ALU_ADC, 0, REG_ITMP2);
1059                                         emit_neg_reg(cd, REG_ITMP2);
1060
1061                                         emit_alu_imm_reg(cd, ALU_AND, iptr->sx.val.l, REG_ITMP1);
1062                                         emit_alu_imm_reg(cd, ALU_AND, iptr->sx.val.l >> 32, REG_ITMP2);
1063
1064                                         emit_neg_reg(cd, REG_ITMP1);
1065                                         emit_alu_imm_reg(cd, ALU_ADC, 0, REG_ITMP2);
1066                                         emit_neg_reg(cd, REG_ITMP2);
1067
1068                                         emit_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst.var->vv.regoff * 8);
1069                                         emit_mov_reg_membase(cd, REG_ITMP2, REG_SP, iptr->dst.var->vv.regoff * 8 + 4);
1070                                 }
1071                         }
1072
1073                         s1 = emit_load_s1(jd, iptr, REG_ITMP12_PACKED);
1074                         d = codegen_reg_of_dst(jd, iptr, REG_RESULT_PACKED);
1075                         M_LNGMOVE(s1, d);
1076                         M_AND_IMM(iptr->sx.val.l, GET_LOW_REG(d));
1077                         M_AND_IMM(iptr->sx.val.l >> 32, GET_HIGH_REG(d));
1078                         M_TEST(GET_LOW_REG(s1));
1079                         M_BGE(0);
1080                         M_LNGMOVE(s1, d);
1081                 break;
1082 #endif
1083
1084                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1085
1086                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1087                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1088                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1089                         M_INTMOVE(s2, ECX);                       /* s2 may be equal to d */
1090                         M_INTMOVE(s1, d);
1091                         M_SLL(d);
1092                         emit_store_dst(jd, iptr, d);
1093                         break;
1094
1095                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1096                                       /* sx.val.i = constant                          */
1097
1098                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1099                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1100                         M_INTMOVE(s1, d);
1101                         M_SLL_IMM(iptr->sx.val.i, d);
1102                         emit_store_dst(jd, iptr, d);
1103                         break;
1104
1105                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1106
1107                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1108                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1109                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1110                         M_INTMOVE(s2, ECX);                       /* s2 may be equal to d */
1111                         M_INTMOVE(s1, d);
1112                         M_SRA(d);
1113                         emit_store_dst(jd, iptr, d);
1114                         break;
1115
1116                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1117                                       /* sx.val.i = constant                          */
1118
1119                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1120                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1121                         M_INTMOVE(s1, d);
1122                         M_SRA_IMM(iptr->sx.val.i, d);
1123                         emit_store_dst(jd, iptr, d);
1124                         break;
1125
1126                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1127
1128                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1129                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1130                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1131                         M_INTMOVE(s2, ECX);                       /* s2 may be equal to d */
1132                         M_INTMOVE(s1, d);
1133                         M_SRL(d);
1134                         emit_store_dst(jd, iptr, d);
1135                         break;
1136
1137                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1138                                       /* sx.val.i = constant                          */
1139
1140                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1141                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1142                         M_INTMOVE(s1, d);
1143                         M_SRL_IMM(iptr->sx.val.i, d);
1144                         emit_store_dst(jd, iptr, d);
1145                         break;
1146
1147                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1148
1149                         s1 = emit_load_s1(jd, iptr, REG_ITMP13_PACKED);
1150                         s2 = emit_load_s2(jd, iptr, ECX);
1151                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP13_PACKED);
1152                         M_LNGMOVE(s1, d);
1153                         M_INTMOVE(s2, ECX);
1154                         M_TEST_IMM(32, ECX);
1155                         M_BEQ(2 + 2);
1156                         M_MOV(GET_LOW_REG(d), GET_HIGH_REG(d));
1157                         M_CLR(GET_LOW_REG(d));
1158                         M_SLLD(GET_LOW_REG(d), GET_HIGH_REG(d));
1159                         M_SLL(GET_LOW_REG(d));
1160                         emit_store_dst(jd, iptr, d);
1161                         break;
1162
1163         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1164                                           /* sx.val.i = constant                          */
1165
1166                         s1 = emit_load_s1(jd, iptr, REG_ITMP12_PACKED);
1167                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP12_PACKED);
1168                         M_LNGMOVE(s1, d);
1169                         if (iptr->sx.val.i & 0x20) {
1170                                 M_MOV(GET_LOW_REG(d), GET_HIGH_REG(d));
1171                                 M_CLR(GET_LOW_REG(d));
1172                                 M_SLLD_IMM(iptr->sx.val.i & 0x3f, GET_LOW_REG(d),
1173                                                    GET_HIGH_REG(d));
1174                         }
1175                         else {
1176                                 M_SLLD_IMM(iptr->sx.val.i & 0x3f, GET_LOW_REG(d),
1177                                                    GET_HIGH_REG(d));
1178                                 M_SLL_IMM(iptr->sx.val.i & 0x3f, GET_LOW_REG(d));
1179                         }
1180                         emit_store_dst(jd, iptr, d);
1181                         break;
1182
1183                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1184
1185                         s1 = emit_load_s1(jd, iptr, REG_ITMP13_PACKED);
1186                         s2 = emit_load_s2(jd, iptr, ECX);
1187                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP13_PACKED);
1188                         M_LNGMOVE(s1, d);
1189                         M_INTMOVE(s2, ECX);
1190                         M_TEST_IMM(32, ECX);
1191                         M_BEQ(2 + 3);
1192                         M_MOV(GET_HIGH_REG(d), GET_LOW_REG(d));
1193                         M_SRA_IMM(31, GET_HIGH_REG(d));
1194                         M_SRLD(GET_HIGH_REG(d), GET_LOW_REG(d));
1195                         M_SRA(GET_HIGH_REG(d));
1196                         emit_store_dst(jd, iptr, d);
1197                         break;
1198
1199                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1200                                       /* sx.val.i = constant                          */
1201
1202                         s1 = emit_load_s1(jd, iptr, REG_ITMP12_PACKED);
1203                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP12_PACKED);
1204                         M_LNGMOVE(s1, d);
1205                         if (iptr->sx.val.i & 0x20) {
1206                                 M_MOV(GET_HIGH_REG(d), GET_LOW_REG(d));
1207                                 M_SRA_IMM(31, GET_HIGH_REG(d));
1208                                 M_SRLD_IMM(iptr->sx.val.i & 0x3f, GET_HIGH_REG(d),
1209                                                    GET_LOW_REG(d));
1210                         }
1211                         else {
1212                                 M_SRLD_IMM(iptr->sx.val.i & 0x3f, GET_HIGH_REG(d),
1213                                                    GET_LOW_REG(d));
1214                                 M_SRA_IMM(iptr->sx.val.i & 0x3f, GET_HIGH_REG(d));
1215                         }
1216                         emit_store_dst(jd, iptr, d);
1217                         break;
1218
1219                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1220
1221                         s1 = emit_load_s1(jd, iptr, REG_ITMP13_PACKED);
1222                         s2 = emit_load_s2(jd, iptr, ECX);
1223                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP13_PACKED);
1224                         M_LNGMOVE(s1, d);
1225                         M_INTMOVE(s2, ECX);
1226                         M_TEST_IMM(32, ECX);
1227                         M_BEQ(2 + 2);
1228                         M_MOV(GET_HIGH_REG(d), GET_LOW_REG(d));
1229                         M_CLR(GET_HIGH_REG(d));
1230                         M_SRLD(GET_HIGH_REG(d), GET_LOW_REG(d));
1231                         M_SRL(GET_HIGH_REG(d));
1232                         emit_store_dst(jd, iptr, d);
1233                         break;
1234
1235                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1236                                       /* sx.val.l = constant                          */
1237
1238                         s1 = emit_load_s1(jd, iptr, REG_ITMP12_PACKED);
1239                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP12_PACKED);
1240                         M_LNGMOVE(s1, d);
1241                         if (iptr->sx.val.i & 0x20) {
1242                                 M_MOV(GET_HIGH_REG(d), GET_LOW_REG(d));
1243                                 M_CLR(GET_HIGH_REG(d));
1244                                 M_SRLD_IMM(iptr->sx.val.i & 0x3f, GET_HIGH_REG(d),
1245                                                    GET_LOW_REG(d));
1246                         }
1247                         else {
1248                                 M_SRLD_IMM(iptr->sx.val.i & 0x3f, GET_HIGH_REG(d),
1249                                                    GET_LOW_REG(d));
1250                                 M_SRL_IMM(iptr->sx.val.i & 0x3f, GET_HIGH_REG(d));
1251                         }
1252                         emit_store_dst(jd, iptr, d);
1253                         break;
1254
1255                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1256
1257                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1258                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1259                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1260                         if (s2 == d)
1261                                 M_AND(s1, d);
1262                         else {
1263                                 M_INTMOVE(s1, d);
1264                                 M_AND(s2, d);
1265                         }
1266                         emit_store_dst(jd, iptr, d);
1267                         break;
1268
1269                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1270                                       /* sx.val.i = constant                          */
1271
1272                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1273                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1274                         M_INTMOVE(s1, d);
1275                         M_AND_IMM(iptr->sx.val.i, d);
1276                         emit_store_dst(jd, iptr, d);
1277                         break;
1278
1279                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1280
1281                         s1 = emit_load_s1_low(jd, iptr, REG_ITMP1);
1282                         s2 = emit_load_s2_low(jd, iptr, REG_ITMP2);
1283                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP12_PACKED);
1284                         if (s2 == GET_LOW_REG(d))
1285                                 M_AND(s1, GET_LOW_REG(d));
1286                         else {
1287                                 M_INTMOVE(s1, GET_LOW_REG(d));
1288                                 M_AND(s2, GET_LOW_REG(d));
1289                         }
1290                         /* REG_ITMP1 probably contains low 32-bit of destination */
1291                         s1 = emit_load_s1_high(jd, iptr, REG_ITMP2);
1292                         s2 = emit_load_s2_high(jd, iptr, REG_ITMP3);
1293                         if (s2 == GET_HIGH_REG(d))
1294                                 M_AND(s1, GET_HIGH_REG(d));
1295                         else {
1296                                 M_INTMOVE(s1, GET_HIGH_REG(d));
1297                                 M_AND(s2, GET_HIGH_REG(d));
1298                         }
1299                         emit_store_dst(jd, iptr, d);
1300                         break;
1301
1302                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1303                                       /* sx.val.l = constant                          */
1304
1305                         s1 = emit_load_s1(jd, iptr, REG_ITMP12_PACKED);
1306                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP12_PACKED);
1307                         M_LNGMOVE(s1, d);
1308                         M_AND_IMM(iptr->sx.val.l, GET_LOW_REG(d));
1309                         M_AND_IMM(iptr->sx.val.l >> 32, GET_HIGH_REG(d));
1310                         emit_store_dst(jd, iptr, d);
1311                         break;
1312
1313                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1314
1315                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1316                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1317                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1318                         if (s2 == d)
1319                                 M_OR(s1, d);
1320                         else {
1321                                 M_INTMOVE(s1, d);
1322                                 M_OR(s2, d);
1323                         }
1324                         emit_store_dst(jd, iptr, d);
1325                         break;
1326
1327                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1328                                       /* sx.val.i = constant                          */
1329
1330                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1331                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1332                         M_INTMOVE(s1, d);
1333                         M_OR_IMM(iptr->sx.val.i, d);
1334                         emit_store_dst(jd, iptr, d);
1335                         break;
1336
1337                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1338
1339                         s1 = emit_load_s1_low(jd, iptr, REG_ITMP1);
1340                         s2 = emit_load_s2_low(jd, iptr, REG_ITMP2);
1341                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP12_PACKED);
1342                         if (s2 == GET_LOW_REG(d))
1343                                 M_OR(s1, GET_LOW_REG(d));
1344                         else {
1345                                 M_INTMOVE(s1, GET_LOW_REG(d));
1346                                 M_OR(s2, GET_LOW_REG(d));
1347                         }
1348                         /* REG_ITMP1 probably contains low 32-bit of destination */
1349                         s1 = emit_load_s1_high(jd, iptr, REG_ITMP2);
1350                         s2 = emit_load_s2_high(jd, iptr, REG_ITMP3);
1351                         if (s2 == GET_HIGH_REG(d))
1352                                 M_OR(s1, GET_HIGH_REG(d));
1353                         else {
1354                                 M_INTMOVE(s1, GET_HIGH_REG(d));
1355                                 M_OR(s2, GET_HIGH_REG(d));
1356                         }
1357                         emit_store_dst(jd, iptr, d);
1358                         break;
1359
1360                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1361                                       /* sx.val.l = constant                          */
1362
1363                         s1 = emit_load_s1(jd, iptr, REG_ITMP12_PACKED);
1364                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP12_PACKED);
1365                         M_LNGMOVE(s1, d);
1366                         M_OR_IMM(iptr->sx.val.l, GET_LOW_REG(d));
1367                         M_OR_IMM(iptr->sx.val.l >> 32, GET_HIGH_REG(d));
1368                         emit_store_dst(jd, iptr, d);
1369                         break;
1370
1371                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1372
1373                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1374                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1375                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1376                         if (s2 == d)
1377                                 M_XOR(s1, d);
1378                         else {
1379                                 M_INTMOVE(s1, d);
1380                                 M_XOR(s2, d);
1381                         }
1382                         emit_store_dst(jd, iptr, d);
1383                         break;
1384
1385                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1386                                       /* sx.val.i = constant                          */
1387
1388                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1389                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1390                         M_INTMOVE(s1, d);
1391                         M_XOR_IMM(iptr->sx.val.i, d);
1392                         emit_store_dst(jd, iptr, d);
1393                         break;
1394
1395                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1396
1397                         s1 = emit_load_s1_low(jd, iptr, REG_ITMP1);
1398                         s2 = emit_load_s2_low(jd, iptr, REG_ITMP2);
1399                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP12_PACKED);
1400                         if (s2 == GET_LOW_REG(d))
1401                                 M_XOR(s1, GET_LOW_REG(d));
1402                         else {
1403                                 M_INTMOVE(s1, GET_LOW_REG(d));
1404                                 M_XOR(s2, GET_LOW_REG(d));
1405                         }
1406                         /* REG_ITMP1 probably contains low 32-bit of destination */
1407                         s1 = emit_load_s1_high(jd, iptr, REG_ITMP2);
1408                         s2 = emit_load_s2_high(jd, iptr, REG_ITMP3);
1409                         if (s2 == GET_HIGH_REG(d))
1410                                 M_XOR(s1, GET_HIGH_REG(d));
1411                         else {
1412                                 M_INTMOVE(s1, GET_HIGH_REG(d));
1413                                 M_XOR(s2, GET_HIGH_REG(d));
1414                         }
1415                         emit_store_dst(jd, iptr, d);
1416                         break;
1417
1418                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1419                                       /* sx.val.l = constant                          */
1420
1421                         s1 = emit_load_s1(jd, iptr, REG_ITMP12_PACKED);
1422                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP12_PACKED);
1423                         M_LNGMOVE(s1, d);
1424                         M_XOR_IMM(iptr->sx.val.l, GET_LOW_REG(d));
1425                         M_XOR_IMM(iptr->sx.val.l >> 32, GET_HIGH_REG(d));
1426                         emit_store_dst(jd, iptr, d);
1427                         break;
1428
1429
1430                 /* floating operations ************************************************/
1431
1432                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1433
1434                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1435                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1436                         emit_fchs(cd);
1437                         emit_store_dst(jd, iptr, d);
1438                         break;
1439
1440                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1441
1442                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1443                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1444                         emit_fchs(cd);
1445                         emit_store_dst(jd, iptr, d);
1446                         break;
1447
1448                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1449
1450                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1451                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1452                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1453                         emit_faddp(cd);
1454                         emit_store_dst(jd, iptr, d);
1455                         break;
1456
1457                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1458
1459                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1460                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1461                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1462                         emit_faddp(cd);
1463                         emit_store_dst(jd, iptr, d);
1464                         break;
1465
1466                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1467
1468                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1469                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1470                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1471                         emit_fsubp(cd);
1472                         emit_store_dst(jd, iptr, d);
1473                         break;
1474
1475                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1476
1477                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1478                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1479                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1480                         emit_fsubp(cd);
1481                         emit_store_dst(jd, iptr, d);
1482                         break;
1483
1484                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1485
1486                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1487                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1488                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1489                         emit_fmulp(cd);
1490                         emit_store_dst(jd, iptr, d);
1491                         break;
1492
1493                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1494
1495                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1496                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1497                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1498                         emit_fmulp(cd);
1499                         emit_store_dst(jd, iptr, d);
1500                         break;
1501
1502                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1503
1504                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1505                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1506                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1507                         emit_fdivp(cd);
1508                         emit_store_dst(jd, iptr, d);
1509                         break;
1510
1511                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1512
1513                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1514                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1515                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1516                         emit_fdivp(cd);
1517                         emit_store_dst(jd, iptr, d);
1518                         break;
1519
1520                 case ICMD_FREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1521
1522                         /* exchanged to skip fxch */
1523                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1524                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1525                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1526 /*                      emit_fxch(cd); */
1527                         emit_fprem(cd);
1528                         emit_wait(cd);
1529                         emit_fnstsw(cd);
1530                         emit_sahf(cd);
1531                         emit_jcc(cd, CC_P, -(2 + 1 + 2 + 1 + 6));
1532                         emit_store_dst(jd, iptr, d);
1533                         emit_ffree_reg(cd, 0);
1534                         emit_fincstp(cd);
1535                         break;
1536
1537                 case ICMD_DREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1538
1539                         /* exchanged to skip fxch */
1540                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1541                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1542                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1543 /*                      emit_fxch(cd); */
1544                         emit_fprem(cd);
1545                         emit_wait(cd);
1546                         emit_fnstsw(cd);
1547                         emit_sahf(cd);
1548                         emit_jcc(cd, CC_P, -(2 + 1 + 2 + 1 + 6));
1549                         emit_store_dst(jd, iptr, d);
1550                         emit_ffree_reg(cd, 0);
1551                         emit_fincstp(cd);
1552                         break;
1553
1554                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1555                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1556
1557                         var = VAROP(iptr->s1);
1558                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1559
1560                         if (var->flags & INMEMORY) {
1561                                 emit_fildl_membase(cd, REG_SP, var->vv.regoff);
1562                         } else {
1563                                 /* XXX not thread safe! */
1564                                 disp = dseg_add_unique_s4(cd, 0);
1565                                 emit_mov_imm_reg(cd, 0, REG_ITMP1);
1566                                 dseg_adddata(cd);
1567                                 emit_mov_reg_membase(cd, var->vv.regoff, REG_ITMP1, disp);
1568                                 emit_fildl_membase(cd, REG_ITMP1, disp);
1569                         }
1570
1571                         emit_store_dst(jd, iptr, d);
1572                         break;
1573
1574                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1575                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1576
1577                         var = VAROP(iptr->s1);
1578                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1579                         if (var->flags & INMEMORY) {
1580                                 emit_fildll_membase(cd, REG_SP, var->vv.regoff);
1581
1582                         } else {
1583                                 log_text("L2F: longs have to be in memory");
1584                                 assert(0);
1585                         }
1586                         emit_store_dst(jd, iptr, d);
1587                         break;
1588
1589                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1590
1591                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1592                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1593
1594                         emit_mov_imm_reg(cd, 0, REG_ITMP1);
1595                         dseg_adddata(cd);
1596
1597                         /* Round to zero, 53-bit mode, exception masked */
1598                         disp = dseg_add_s4(cd, 0x0e7f);
1599                         emit_fldcw_membase(cd, REG_ITMP1, disp);
1600
1601                         var = VAROP(iptr->dst);
1602                         var1 = VAROP(iptr->s1);
1603
1604                         if (var->flags & INMEMORY) {
1605                                 emit_fistpl_membase(cd, REG_SP, var->vv.regoff);
1606
1607                                 /* Round to nearest, 53-bit mode, exceptions masked */
1608                                 disp = dseg_add_s4(cd, 0x027f);
1609                                 emit_fldcw_membase(cd, REG_ITMP1, disp);
1610
1611                                 emit_alu_imm_membase(cd, ALU_CMP, 0x80000000,
1612                                                                          REG_SP, var->vv.regoff);
1613
1614                                 disp = 3;
1615                                 CALCOFFSETBYTES(disp, REG_SP, var1->vv.regoff);
1616                                 disp += 5 + 2 + 3;
1617                                 CALCOFFSETBYTES(disp, REG_SP, var->vv.regoff);
1618
1619                         } else {
1620                                 /* XXX not thread safe! */
1621                                 disp = dseg_add_unique_s4(cd, 0);
1622                                 emit_fistpl_membase(cd, REG_ITMP1, disp);
1623                                 emit_mov_membase_reg(cd, REG_ITMP1, disp, var->vv.regoff);
1624
1625                                 /* Round to nearest, 53-bit mode, exceptions masked */
1626                                 disp = dseg_add_s4(cd, 0x027f);
1627                                 emit_fldcw_membase(cd, REG_ITMP1, disp);
1628
1629                                 emit_alu_imm_reg(cd, ALU_CMP, 0x80000000, var->vv.regoff);
1630
1631                                 disp = 3;
1632                                 CALCOFFSETBYTES(disp, REG_SP, var1->vv.regoff);
1633                                 disp += 5 + 2 + ((REG_RESULT == var->vv.regoff) ? 0 : 2);
1634                         }
1635
1636                         emit_jcc(cd, CC_NE, disp);
1637
1638                         /* XXX: change this when we use registers */
1639                         emit_flds_membase(cd, REG_SP, var1->vv.regoff);
1640                         emit_mov_imm_reg(cd, (ptrint) asm_builtin_f2i, REG_ITMP1);
1641                         emit_call_reg(cd, REG_ITMP1);
1642
1643                         if (var->flags & INMEMORY) {
1644                                 emit_mov_reg_membase(cd, REG_RESULT, REG_SP, var->vv.regoff);
1645
1646                         } else {
1647                                 M_INTMOVE(REG_RESULT, var->vv.regoff);
1648                         }
1649                         break;
1650
1651                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1652
1653                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1654                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1655
1656                         emit_mov_imm_reg(cd, 0, REG_ITMP1);
1657                         dseg_adddata(cd);
1658
1659                         /* Round to zero, 53-bit mode, exception masked */
1660                         disp = dseg_add_s4(cd, 0x0e7f);
1661                         emit_fldcw_membase(cd, REG_ITMP1, disp);
1662
1663                         var  = VAROP(iptr->dst);
1664                         var1 = VAROP(iptr->s1);
1665
1666                         if (var->flags & INMEMORY) {
1667                                 emit_fistpl_membase(cd, REG_SP, var->vv.regoff);
1668
1669                                 /* Round to nearest, 53-bit mode, exceptions masked */
1670                                 disp = dseg_add_s4(cd, 0x027f);
1671                                 emit_fldcw_membase(cd, REG_ITMP1, disp);
1672
1673                                 emit_alu_imm_membase(cd, ALU_CMP, 0x80000000,
1674                                                                          REG_SP, var->vv.regoff);
1675
1676                                 disp = 3;
1677                                 CALCOFFSETBYTES(disp, REG_SP, var1->vv.regoff);
1678                                 disp += 5 + 2 + 3;
1679                                 CALCOFFSETBYTES(disp, REG_SP, var->vv.regoff);
1680
1681                         } else {
1682                                 /* XXX not thread safe! */
1683                                 disp = dseg_add_unique_s4(cd, 0);
1684                                 emit_fistpl_membase(cd, REG_ITMP1, disp);
1685                                 emit_mov_membase_reg(cd, REG_ITMP1, disp, var->vv.regoff);
1686
1687                                 /* Round to nearest, 53-bit mode, exceptions masked */
1688                                 disp = dseg_add_s4(cd, 0x027f);
1689                                 emit_fldcw_membase(cd, REG_ITMP1, disp);
1690
1691                                 emit_alu_imm_reg(cd, ALU_CMP, 0x80000000, var->vv.regoff);
1692
1693                                 disp = 3;
1694                                 CALCOFFSETBYTES(disp, REG_SP, var1->vv.regoff);
1695                                 disp += 5 + 2 + ((REG_RESULT == var->vv.regoff) ? 0 : 2);
1696                         }
1697
1698                         emit_jcc(cd, CC_NE, disp);
1699
1700                         /* XXX: change this when we use registers */
1701                         emit_fldl_membase(cd, REG_SP, var1->vv.regoff);
1702                         emit_mov_imm_reg(cd, (ptrint) asm_builtin_d2i, REG_ITMP1);
1703                         emit_call_reg(cd, REG_ITMP1);
1704
1705                         if (var->flags & INMEMORY) {
1706                                 emit_mov_reg_membase(cd, REG_RESULT, REG_SP, var->vv.regoff);
1707                         } else {
1708                                 M_INTMOVE(REG_RESULT, var->vv.regoff);
1709                         }
1710                         break;
1711
1712                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1713
1714                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1715                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1716
1717                         emit_mov_imm_reg(cd, 0, REG_ITMP1);
1718                         dseg_adddata(cd);
1719
1720                         /* Round to zero, 53-bit mode, exception masked */
1721                         disp = dseg_add_s4(cd, 0x0e7f);
1722                         emit_fldcw_membase(cd, REG_ITMP1, disp);
1723
1724                         var  = VAROP(iptr->dst);
1725                         var1 = VAROP(iptr->s1);
1726
1727                         if (var->flags & INMEMORY) {
1728                                 emit_fistpll_membase(cd, REG_SP, var->vv.regoff);
1729
1730                                 /* Round to nearest, 53-bit mode, exceptions masked */
1731                                 disp = dseg_add_s4(cd, 0x027f);
1732                                 emit_fldcw_membase(cd, REG_ITMP1, disp);
1733
1734                                 emit_alu_imm_membase(cd, ALU_CMP, 0x80000000,
1735                                                                          REG_SP, var->vv.regoff + 4);
1736
1737                                 disp = 6 + 4;
1738                                 CALCOFFSETBYTES(disp, REG_SP, var->vv.regoff);
1739                                 disp += 3;
1740                                 CALCOFFSETBYTES(disp, REG_SP, var1->vv.regoff);
1741                                 disp += 5 + 2;
1742                                 disp += 3;
1743                                 CALCOFFSETBYTES(disp, REG_SP, var->vv.regoff);
1744                                 disp += 3;
1745                                 CALCOFFSETBYTES(disp, REG_SP, var->vv.regoff + 4);
1746
1747                                 emit_jcc(cd, CC_NE, disp);
1748
1749                                 emit_alu_imm_membase(cd, ALU_CMP, 0,
1750                                                                          REG_SP, var->vv.regoff);
1751
1752                                 disp = 3;
1753                                 CALCOFFSETBYTES(disp, REG_SP, var1->vv.regoff);
1754                                 disp += 5 + 2 + 3;
1755                                 CALCOFFSETBYTES(disp, REG_SP, var->vv.regoff);
1756
1757                                 emit_jcc(cd, CC_NE, disp);
1758
1759                                 /* XXX: change this when we use registers */
1760                                 emit_flds_membase(cd, REG_SP, var1->vv.regoff);
1761                                 emit_mov_imm_reg(cd, (ptrint) asm_builtin_f2l, REG_ITMP1);
1762                                 emit_call_reg(cd, REG_ITMP1);
1763                                 emit_mov_reg_membase(cd, REG_RESULT, REG_SP, var->vv.regoff);
1764                                 emit_mov_reg_membase(cd, REG_RESULT2,
1765                                                                          REG_SP, var->vv.regoff + 4);
1766
1767                         } else {
1768                                 log_text("F2L: longs have to be in memory");
1769                                 assert(0);
1770                         }
1771                         break;
1772
1773                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1774
1775                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1776                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1777
1778                         emit_mov_imm_reg(cd, 0, REG_ITMP1);
1779                         dseg_adddata(cd);
1780
1781                         /* Round to zero, 53-bit mode, exception masked */
1782                         disp = dseg_add_s4(cd, 0x0e7f);
1783                         emit_fldcw_membase(cd, REG_ITMP1, disp);
1784
1785                         var  = VAROP(iptr->dst);
1786                         var1 = VAROP(iptr->s1);
1787
1788                         if (var->flags & INMEMORY) {
1789                                 emit_fistpll_membase(cd, REG_SP, var->vv.regoff);
1790
1791                                 /* Round to nearest, 53-bit mode, exceptions masked */
1792                                 disp = dseg_add_s4(cd, 0x027f);
1793                                 emit_fldcw_membase(cd, REG_ITMP1, disp);
1794
1795                                 emit_alu_imm_membase(cd, ALU_CMP, 0x80000000,
1796                                                                          REG_SP, var->vv.regoff + 4);
1797
1798                                 disp = 6 + 4;
1799                                 CALCOFFSETBYTES(disp, REG_SP, var->vv.regoff);
1800                                 disp += 3;
1801                                 CALCOFFSETBYTES(disp, REG_SP, var1->vv.regoff);
1802                                 disp += 5 + 2;
1803                                 disp += 3;
1804                                 CALCOFFSETBYTES(disp, REG_SP, var->vv.regoff);
1805                                 disp += 3;
1806                                 CALCOFFSETBYTES(disp, REG_SP, var->vv.regoff + 4);
1807
1808                                 emit_jcc(cd, CC_NE, disp);
1809
1810                                 emit_alu_imm_membase(cd, ALU_CMP, 0, REG_SP, var->vv.regoff);
1811
1812                                 disp = 3;
1813                                 CALCOFFSETBYTES(disp, REG_SP, var1->vv.regoff);
1814                                 disp += 5 + 2 + 3;
1815                                 CALCOFFSETBYTES(disp, REG_SP, var->vv.regoff);
1816
1817                                 emit_jcc(cd, CC_NE, disp);
1818
1819                                 /* XXX: change this when we use registers */
1820                                 emit_fldl_membase(cd, REG_SP, var1->vv.regoff);
1821                                 emit_mov_imm_reg(cd, (ptrint) asm_builtin_d2l, REG_ITMP1);
1822                                 emit_call_reg(cd, REG_ITMP1);
1823                                 emit_mov_reg_membase(cd, REG_RESULT, REG_SP, var->vv.regoff);
1824                                 emit_mov_reg_membase(cd, REG_RESULT2,
1825                                                                          REG_SP, var->vv.regoff + 4);
1826
1827                         } else {
1828                                 log_text("D2L: longs have to be in memory");
1829                                 assert(0);
1830                         }
1831                         break;
1832
1833                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1834
1835                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1836                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1837                         /* nothing to do */
1838                         emit_store_dst(jd, iptr, d);
1839                         break;
1840
1841                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1842
1843                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1844                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1845                         /* nothing to do */
1846                         emit_store_dst(jd, iptr, d);
1847                         break;
1848
1849                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1850                 case ICMD_DCMPL:
1851
1852                         /* exchanged to skip fxch */
1853                         s2 = emit_load_s1(jd, iptr, REG_FTMP1);
1854                         s1 = emit_load_s2(jd, iptr, REG_FTMP2);
1855                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1856 /*                      emit_fxch(cd); */
1857                         emit_fucompp(cd);
1858                         emit_fnstsw(cd);
1859                         emit_test_imm_reg(cd, 0x400, EAX);    /* unordered treat as GT */
1860                         emit_jcc(cd, CC_E, 6);
1861                         emit_alu_imm_reg(cd, ALU_AND, 0x000000ff, EAX);
1862                         emit_sahf(cd);
1863                         emit_mov_imm_reg(cd, 0, d);    /* does not affect flags */
1864                         emit_jcc(cd, CC_E, 6 + 3 + 5 + 3);
1865                         emit_jcc(cd, CC_B, 3 + 5);
1866                         emit_alu_imm_reg(cd, ALU_SUB, 1, d);
1867                         emit_jmp_imm(cd, 3);
1868                         emit_alu_imm_reg(cd, ALU_ADD, 1, d);
1869                         emit_store_dst(jd, iptr, d);
1870                         break;
1871
1872                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1873                 case ICMD_DCMPG:
1874
1875                         /* exchanged to skip fxch */
1876                         s2 = emit_load_s1(jd, iptr, REG_FTMP1);
1877                         s1 = emit_load_s2(jd, iptr, REG_FTMP2);
1878                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1879 /*                      emit_fxch(cd); */
1880                         emit_fucompp(cd);
1881                         emit_fnstsw(cd);
1882                         emit_test_imm_reg(cd, 0x400, EAX);    /* unordered treat as LT */
1883                         emit_jcc(cd, CC_E, 3);
1884                         emit_movb_imm_reg(cd, 1, REG_AH);
1885                         emit_sahf(cd);
1886                         emit_mov_imm_reg(cd, 0, d);    /* does not affect flags */
1887                         emit_jcc(cd, CC_E, 6 + 3 + 5 + 3);
1888                         emit_jcc(cd, CC_B, 3 + 5);
1889                         emit_alu_imm_reg(cd, ALU_SUB, 1, d);
1890                         emit_jmp_imm(cd, 3);
1891                         emit_alu_imm_reg(cd, ALU_ADD, 1, d);
1892                         emit_store_dst(jd, iptr, d);
1893                         break;
1894
1895
1896                 /* memory operations **************************************************/
1897
1898                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., length              */
1899
1900                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1901                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1902                         /* implicit null-pointer check */
1903                         M_ILD(d, s1, OFFSET(java_array_t, size));
1904                         emit_store_dst(jd, iptr, d);
1905                         break;
1906
1907                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1908
1909                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1910                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1911                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1912                         /* implicit null-pointer check */
1913                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1914                         emit_movsbl_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]),
1915                                                                          s1, s2, 0, d);
1916                         emit_store_dst(jd, iptr, d);
1917                         break;
1918
1919                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1920
1921                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1922                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1923                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1924                         /* implicit null-pointer check */
1925                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1926                         emit_movzwl_memindex_reg(cd, OFFSET(java_chararray_t, data[0]),
1927                                                                          s1, s2, 1, d);
1928                         emit_store_dst(jd, iptr, d);
1929                         break;
1930
1931                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1932
1933                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1934                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1935                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1936                         /* implicit null-pointer check */
1937                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1938                         emit_movswl_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]),
1939                                                                          s1, s2, 1, d);
1940                         emit_store_dst(jd, iptr, d);
1941                         break;
1942
1943                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1944
1945                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1946                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1947                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1948                         /* implicit null-pointer check */
1949                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1950                         emit_mov_memindex_reg(cd, OFFSET(java_intarray_t, data[0]),
1951                                                                   s1, s2, 2, d);
1952                         emit_store_dst(jd, iptr, d);
1953                         break;
1954
1955                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1956
1957                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1958                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1959                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1960                         /* implicit null-pointer check */
1961                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1962
1963                         var = VAROP(iptr->dst);
1964
1965                         assert(var->flags & INMEMORY);
1966                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]),
1967                                                                   s1, s2, 3, REG_ITMP3);
1968                         emit_mov_reg_membase(cd, REG_ITMP3, REG_SP, var->vv.regoff);
1969                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]) + 4,
1970                                                                   s1, s2, 3, REG_ITMP3);
1971                         emit_mov_reg_membase(cd, REG_ITMP3, REG_SP, var->vv.regoff + 4);
1972                         break;
1973
1974                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1975
1976                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1977                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1978                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1979                         /* implicit null-pointer check */
1980                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1981                         emit_flds_memindex(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1982                         emit_store_dst(jd, iptr, d);
1983                         break;
1984
1985                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1986
1987                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1988                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1989                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1990                         /* implicit null-pointer check */
1991                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1992                         emit_fldl_memindex(cd, OFFSET(java_doublearray_t, data[0]), s1, s2,3);
1993                         emit_store_dst(jd, iptr, d);
1994                         break;
1995
1996                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1997
1998                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1999                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2000                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
2001                         /* implicit null-pointer check */
2002                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
2003                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]),
2004                                                                   s1, s2, 2, d);
2005                         emit_store_dst(jd, iptr, d);
2006                         break;
2007
2008
2009                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2010
2011                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2012                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2013                         /* implicit null-pointer check */
2014                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
2015                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
2016                         if (s3 >= EBP) {
2017                                 /* because EBP, ESI, EDI have no xH and xL nibbles */
2018                                 M_INTMOVE(s3, REG_ITMP3);
2019                                 s3 = REG_ITMP3;
2020                         }
2021                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]),
2022                                                                    s1, s2, 0);
2023                         break;
2024
2025                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2026
2027                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2028                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2029                         /* implicit null-pointer check */
2030                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
2031                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
2032                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]),
2033                                                                    s1, s2, 1);
2034                         break;
2035
2036                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2037
2038                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2039                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2040                         /* implicit null-pointer check */
2041                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
2042                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
2043                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]),
2044                                                                    s1, s2, 1);
2045                         break;
2046
2047                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2048
2049                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2050                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2051                         /* implicit null-pointer check */
2052                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
2053                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
2054                         emit_mov_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]),
2055                                                                   s1, s2, 2);
2056                         break;
2057
2058                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2059
2060                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2061                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2062                         /* implicit null-pointer check */
2063                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
2064
2065                         var = VAROP(iptr->sx.s23.s3);
2066
2067                         assert(var->flags & INMEMORY);
2068                         emit_mov_membase_reg(cd, REG_SP, var->vv.regoff, REG_ITMP3);
2069                         emit_mov_reg_memindex(cd, REG_ITMP3, OFFSET(java_longarray_t, data[0])
2070                                                                   , s1, s2, 3);
2071                         emit_mov_membase_reg(cd, REG_SP, var->vv.regoff + 4, REG_ITMP3);
2072                         emit_mov_reg_memindex(cd, REG_ITMP3,
2073                                                             OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
2074                         break;
2075
2076                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2077
2078                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2079                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2080                         /* implicit null-pointer check */
2081                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
2082                         s3 = emit_load_s3(jd, iptr, REG_FTMP1);
2083                         emit_fstps_memindex(cd, OFFSET(java_floatarray_t, data[0]), s1, s2,2);
2084                         break;
2085
2086                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2087
2088                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2089                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2090                         /* implicit null-pointer check */
2091                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
2092                         s3 = emit_load_s3(jd, iptr, REG_FTMP1);
2093                         emit_fstpl_memindex(cd, OFFSET(java_doublearray_t, data[0]),
2094                                                                 s1, s2, 3);
2095                         break;
2096
2097                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2098
2099                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2100                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2101                         /* implicit null-pointer check */
2102                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
2103                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
2104
2105                         M_AST(s1, REG_SP, 0 * 4);
2106                         M_AST(s3, REG_SP, 1 * 4);
2107                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
2108                         M_CALL(REG_ITMP1);
2109                         emit_arraystore_check(cd, iptr);
2110
2111                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2112                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2113                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
2114                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]),
2115                                                                   s1, s2, 2);
2116                         break;
2117
2118                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2119
2120                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2121                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2122                         /* implicit null-pointer check */
2123                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
2124                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval,
2125                                                                    OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
2126                         break;
2127
2128                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2129
2130                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2131                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2132                         /* implicit null-pointer check */
2133                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
2134                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval,
2135                                                                    OFFSET(java_chararray_t, data[0]), s1, s2, 1);
2136                         break;
2137
2138                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2139
2140                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2141                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2142                         /* implicit null-pointer check */
2143                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
2144                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval,
2145                                                                    OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
2146                         break;
2147
2148                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2149
2150                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2151                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2152                         /* implicit null-pointer check */
2153                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
2154                         emit_mov_imm_memindex(cd, iptr->sx.s23.s3.constval,
2155                                                                   OFFSET(java_intarray_t, data[0]), s1, s2, 2);
2156                         break;
2157
2158                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2159
2160                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2161                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2162                         /* implicit null-pointer check */
2163                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
2164                         emit_mov_imm_memindex(cd,
2165                                                    (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff),
2166                                                    OFFSET(java_longarray_t, data[0]), s1, s2, 3);
2167                         emit_mov_imm_memindex(cd,
2168                                                         ((s4)iptr->sx.s23.s3.constval) >> 31,
2169                                                         OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
2170                         break;
2171
2172                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2173
2174                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2175                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2176                         /* implicit null-pointer check */
2177                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
2178                         emit_mov_imm_memindex(cd, 0,
2179                                                                   OFFSET(java_objectarray_t, data[0]), s1, s2, 2);
2180                         break;
2181
2182
2183                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2184
2185                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2186                                 uf        = iptr->sx.s23.s3.uf;
2187                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2188                                 disp      = 0;
2189
2190                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, 0);
2191
2192                         }
2193                         else {
2194                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2195                                 fieldtype = fi->type;
2196                                 disp      = (intptr_t) fi->value;
2197
2198                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class))
2199                                         patcher_add_patch_ref(jd, PATCHER_initialize_class, fi->class, 0);
2200                         }
2201
2202                         M_MOV_IMM(disp, REG_ITMP1);
2203                         switch (fieldtype) {
2204                         case TYPE_INT:
2205                         case TYPE_ADR:
2206                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2207                                 M_ILD(d, REG_ITMP1, 0);
2208                                 break;
2209                         case TYPE_LNG:
2210                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP23_PACKED);
2211                                 M_LLD(d, REG_ITMP1, 0);
2212                                 break;
2213                         case TYPE_FLT:
2214                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2215                                 M_FLD(d, REG_ITMP1, 0);
2216                                 break;
2217                         case TYPE_DBL:
2218                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2219                                 M_DLD(d, REG_ITMP1, 0);
2220                                 break;
2221                         }
2222                         emit_store_dst(jd, iptr, d);
2223                         break;
2224
2225                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2226
2227                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2228                                 uf        = iptr->sx.s23.s3.uf;
2229                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2230                                 disp      = 0;
2231
2232                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, 0);
2233                         }
2234                         else {
2235                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2236                                 fieldtype = fi->type;
2237                                 disp      = (intptr_t) fi->value;
2238
2239                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class))
2240                                         patcher_add_patch_ref(jd, PATCHER_initialize_class, fi->class, 0);
2241                         }
2242
2243                         M_MOV_IMM(disp, REG_ITMP1);
2244                         switch (fieldtype) {
2245                         case TYPE_INT:
2246                         case TYPE_ADR:
2247                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2248                                 M_IST(s1, REG_ITMP1, 0);
2249                                 break;
2250                         case TYPE_LNG:
2251                                 s1 = emit_load_s1(jd, iptr, REG_ITMP23_PACKED);
2252                                 M_LST(s1, REG_ITMP1, 0);
2253                                 break;
2254                         case TYPE_FLT:
2255                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
2256                                 emit_fstps_membase(cd, REG_ITMP1, 0);
2257                                 break;
2258                         case TYPE_DBL:
2259                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
2260                                 emit_fstpl_membase(cd, REG_ITMP1, 0);
2261                                 break;
2262                         }
2263                         break;
2264
2265                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2266                                           /* val = value (in current instruction)     */
2267                                           /* following NOP)                           */
2268
2269                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2270                                 uf        = iptr->sx.s23.s3.uf;
2271                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2272                                 disp      = 0;
2273
2274                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, 0);
2275                         }
2276                         else {
2277                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2278                                 fieldtype = fi->type;
2279                                 disp      = (intptr_t) fi->value;
2280
2281                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class))
2282                                         patcher_add_patch_ref(jd, PATCHER_initialize_class, fi->class, 0);
2283                         }
2284
2285                         M_MOV_IMM(disp, REG_ITMP1);
2286                         switch (fieldtype) {
2287                         case TYPE_INT:
2288                         case TYPE_ADR:
2289                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2290                                 break;
2291                         case TYPE_LNG:
2292                                 M_IST_IMM(iptr->sx.s23.s2.constval & 0xffffffff, REG_ITMP1, 0);
2293                                 M_IST_IMM(((s4)iptr->sx.s23.s2.constval) >> 31, REG_ITMP1, 4);
2294                                 break;
2295                         default:
2296                                 assert(0);
2297                         }
2298                         break;
2299
2300                 case ICMD_GETFIELD:   /* .., objectref.  ==> ..., value               */
2301
2302                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2303                         emit_nullpointer_check(cd, iptr, s1);
2304
2305                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2306                                 uf        = iptr->sx.s23.s3.uf;
2307                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2308                                 disp      = 0;
2309
2310                                 patcher_add_patch_ref(jd, PATCHER_getfield,
2311                                                                         iptr->sx.s23.s3.uf, 0);
2312                         }
2313                         else {
2314                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2315                                 fieldtype = fi->type;
2316                                 disp      = fi->offset;
2317                         }
2318
2319                         switch (fieldtype) {
2320                         case TYPE_INT:
2321                         case TYPE_ADR:
2322                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2323                                 M_ILD32(d, s1, disp);
2324                                 break;
2325                         case TYPE_LNG:
2326                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP23_PACKED);
2327                                 M_LLD32(d, s1, disp);
2328                                 break;
2329                         case TYPE_FLT:
2330                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2331                                 M_FLD32(d, s1, disp);
2332                                 break;
2333                         case TYPE_DBL:
2334                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2335                                 M_DLD32(d, s1, disp);
2336                                 break;
2337                         }
2338                         emit_store_dst(jd, iptr, d);
2339                         break;
2340
2341                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2342
2343                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2344                         emit_nullpointer_check(cd, iptr, s1);
2345
2346                         /* must be done here because of code patching */
2347
2348                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2349                                 uf        = iptr->sx.s23.s3.uf;
2350                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2351                         }
2352                         else {
2353                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2354                                 fieldtype = fi->type;
2355                         }
2356
2357                         if (!IS_FLT_DBL_TYPE(fieldtype)) {
2358                                 if (IS_2_WORD_TYPE(fieldtype))
2359                                         s2 = emit_load_s2(jd, iptr, REG_ITMP23_PACKED);
2360                                 else
2361                                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2362                         }
2363                         else
2364                                 s2 = emit_load_s2(jd, iptr, REG_FTMP2);
2365
2366                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2367                                 /* XXX */
2368                                 uf   = iptr->sx.s23.s3.uf;
2369                                 disp = 0;
2370
2371                                 patcher_add_patch_ref(jd, PATCHER_putfield, uf, 0);
2372                         }
2373                         else {
2374                                 /* XXX */
2375                                 fi   = iptr->sx.s23.s3.fmiref->p.field;
2376                                 disp = fi->offset;
2377                         }
2378
2379                         switch (fieldtype) {
2380                         case TYPE_INT:
2381                         case TYPE_ADR:
2382                                 M_IST32(s2, s1, disp);
2383                                 break;
2384                         case TYPE_LNG:
2385                                 M_LST32(s2, s1, disp);
2386                                 break;
2387                         case TYPE_FLT:
2388                                 emit_fstps_membase32(cd, s1, disp);
2389                                 break;
2390                         case TYPE_DBL:
2391                                 emit_fstpl_membase32(cd, s1, disp);
2392                                 break;
2393                         }
2394                         break;
2395
2396                 case ICMD_PUTFIELDCONST:  /* ..., objectref  ==> ...                  */
2397                                           /* val = value (in current instruction)     */
2398                                           /* following NOP)                           */
2399
2400                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2401                         emit_nullpointer_check(cd, iptr, s1);
2402
2403                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2404                                 uf        = iptr->sx.s23.s3.uf;
2405                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2406                                 disp      = 0;
2407
2408                                 patcher_add_patch_ref(jd, PATCHER_putfieldconst,
2409                                                                         uf, 0);
2410                         }
2411                         else {
2412                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2413                                 fieldtype = fi->type;
2414                                 disp      = fi->offset;
2415                         }
2416
2417                         switch (fieldtype) {
2418                         case TYPE_INT:
2419                         case TYPE_ADR:
2420                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2421                                 break;
2422                         case TYPE_LNG:
2423                                 M_IST32_IMM(iptr->sx.s23.s2.constval & 0xffffffff, s1, disp);
2424                                 M_IST32_IMM(((s4)iptr->sx.s23.s2.constval) >> 31, s1, disp + 4);
2425                                 break;
2426                         default:
2427                                 assert(0);
2428                         }
2429                         break;
2430
2431
2432                 /* branch operations **************************************************/
2433
2434                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2435
2436                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2437                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2438
2439 #ifdef ENABLE_VERIFIER
2440                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2441                                 patcher_add_patch_ref(jd, PATCHER_resolve_class,
2442                                                                         iptr->sx.s23.s2.uc, 0);
2443                         }
2444 #endif /* ENABLE_VERIFIER */
2445
2446                         M_CALL_IMM(0);                            /* passing exception pc */
2447                         M_POP(REG_ITMP2_XPC);
2448
2449                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2450                         M_JMP(REG_ITMP3);
2451                         break;
2452
2453                 case ICMD_GOTO:         /* ... ==> ...                                */
2454                 case ICMD_RET:          /* ... ==> ...                                */
2455
2456 #if defined(ENABLE_SSA)
2457                         if ( ls != NULL ) {
2458                                 last_cmd_was_goto = true;
2459
2460                                 /* In case of a Goto phimoves have to be inserted before the */
2461                                 /* jump */
2462
2463                                 codegen_emit_phi_moves(jd, bptr);
2464                         }
2465 #endif
2466                         emit_br(cd, iptr->dst.block);
2467                         ALIGNCODENOP;
2468                         break;
2469
2470                 case ICMD_JSR:          /* ... ==> ...                                */
2471
2472                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2473                         ALIGNCODENOP;
2474                         break;
2475
2476                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2477                 case ICMD_IFNONNULL:
2478
2479                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2480                         M_TEST(s1);
2481                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2482                         break;
2483
2484                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2485                 case ICMD_IFLT:
2486                 case ICMD_IFLE:
2487                 case ICMD_IFNE:
2488                 case ICMD_IFGT:
2489                 case ICMD_IFGE:
2490
2491                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2492                         M_CMP_IMM(iptr->sx.val.i, s1);
2493                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2494                         break;
2495
2496                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2497
2498                         s1 = emit_load_s1(jd, iptr, REG_ITMP12_PACKED);
2499                         if (iptr->sx.val.l == 0) {
2500                                 M_INTMOVE(GET_LOW_REG(s1), REG_ITMP1);
2501                                 M_OR(GET_HIGH_REG(s1), REG_ITMP1);
2502                         }
2503                         else {
2504                                 M_LNGMOVE(s1, REG_ITMP12_PACKED);
2505                                 M_XOR_IMM(iptr->sx.val.l, REG_ITMP1);
2506                                 M_XOR_IMM(iptr->sx.val.l >> 32, REG_ITMP2);
2507                                 M_OR(REG_ITMP2, REG_ITMP1);
2508                         }
2509                         emit_beq(cd, iptr->dst.block);
2510                         break;
2511
2512                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2513
2514                         if (iptr->sx.val.l == 0) {
2515                                 /* If high 32-bit are less than zero, then the 64-bits
2516                                    are too. */
2517                                 s1 = emit_load_s1_high(jd, iptr, REG_ITMP2);
2518                                 M_CMP_IMM(0, s1);
2519                                 emit_blt(cd, iptr->dst.block);
2520                         }
2521                         else {
2522                                 s1 = emit_load_s1(jd, iptr, REG_ITMP12_PACKED);
2523                                 M_CMP_IMM(iptr->sx.val.l >> 32, GET_HIGH_REG(s1));
2524                                 emit_blt(cd, iptr->dst.block);
2525                                 M_BGT(6 + 6);
2526                                 M_CMP_IMM32(iptr->sx.val.l, GET_LOW_REG(s1));
2527                                 emit_bult(cd, iptr->dst.block);
2528                         }
2529                         break;
2530
2531                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2532
2533                         s1 = emit_load_s1(jd, iptr, REG_ITMP12_PACKED);
2534                         M_CMP_IMM(iptr->sx.val.l >> 32, GET_HIGH_REG(s1));
2535                         emit_blt(cd, iptr->dst.block);
2536                         M_BGT(6 + 6);
2537                         M_CMP_IMM32(iptr->sx.val.l, GET_LOW_REG(s1));
2538                         emit_bule(cd, iptr->dst.block);
2539                         break;
2540
2541                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2542
2543                         s1 = emit_load_s1(jd, iptr, REG_ITMP12_PACKED);
2544                         if (iptr->sx.val.l == 0) {
2545                                 M_INTMOVE(GET_LOW_REG(s1), REG_ITMP1);
2546                                 M_OR(GET_HIGH_REG(s1), REG_ITMP1);
2547                         }
2548                         else {
2549                                 M_LNGMOVE(s1, REG_ITMP12_PACKED);
2550                                 M_XOR_IMM(iptr->sx.val.l, REG_ITMP1);
2551                                 M_XOR_IMM(iptr->sx.val.l >> 32, REG_ITMP2);
2552                                 M_OR(REG_ITMP2, REG_ITMP1);
2553                         }
2554                         emit_bne(cd, iptr->dst.block);
2555                         break;
2556
2557                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2558
2559                         s1 = emit_load_s1(jd, iptr, REG_ITMP12_PACKED);
2560                         M_CMP_IMM(iptr->sx.val.l >> 32, GET_HIGH_REG(s1));
2561                         emit_bgt(cd, iptr->dst.block);
2562                         M_BLT(6 + 6);
2563                         M_CMP_IMM32(iptr->sx.val.l, GET_LOW_REG(s1));
2564                         emit_bugt(cd, iptr->dst.block);
2565                         break;
2566
2567                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2568
2569                         if (iptr->sx.val.l == 0) {
2570                                 /* If high 32-bit are greater equal zero, then the
2571                                    64-bits are too. */
2572                                 s1 = emit_load_s1_high(jd, iptr, REG_ITMP2);
2573                                 M_CMP_IMM(0, s1);
2574                                 emit_bge(cd, iptr->dst.block);
2575                         }
2576                         else {
2577                                 s1 = emit_load_s1(jd, iptr, REG_ITMP12_PACKED);
2578                                 M_CMP_IMM(iptr->sx.val.l >> 32, GET_HIGH_REG(s1));
2579                                 emit_bgt(cd, iptr->dst.block);
2580                                 M_BLT(6 + 6);
2581                                 M_CMP_IMM32(iptr->sx.val.l, GET_LOW_REG(s1));
2582                                 emit_buge(cd, iptr->dst.block);
2583                         }
2584                         break;
2585
2586                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2587                 case ICMD_IF_ICMPNE:
2588                 case ICMD_IF_ICMPLT:
2589                 case ICMD_IF_ICMPGT:
2590                 case ICMD_IF_ICMPGE:
2591                 case ICMD_IF_ICMPLE:
2592
2593                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2594                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2595                         M_CMP(s2, s1);
2596                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2597                         break;
2598
2599                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2600                 case ICMD_IF_ACMPNE:
2601
2602                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2603                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2604                         M_CMP(s2, s1);
2605                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2606                         break;
2607
2608                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2609
2610                         s1 = emit_load_s1_low(jd, iptr, REG_ITMP1);
2611                         s2 = emit_load_s2_low(jd, iptr, REG_ITMP2);
2612                         M_INTMOVE(s1, REG_ITMP1);
2613                         M_XOR(s2, REG_ITMP1);
2614                         s1 = emit_load_s1_high(jd, iptr, REG_ITMP2);
2615                         s2 = emit_load_s2_high(jd, iptr, REG_ITMP3);
2616                         M_INTMOVE(s1, REG_ITMP2);
2617                         M_XOR(s2, REG_ITMP2);
2618                         M_OR(REG_ITMP1, REG_ITMP2);
2619                         emit_beq(cd, iptr->dst.block);
2620                         break;
2621
2622                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2623
2624                         s1 = emit_load_s1_low(jd, iptr, REG_ITMP1);
2625                         s2 = emit_load_s2_low(jd, iptr, REG_ITMP2);
2626                         M_INTMOVE(s1, REG_ITMP1);
2627                         M_XOR(s2, REG_ITMP1);
2628                         s1 = emit_load_s1_high(jd, iptr, REG_ITMP2);
2629                         s2 = emit_load_s2_high(jd, iptr, REG_ITMP3);
2630                         M_INTMOVE(s1, REG_ITMP2);
2631                         M_XOR(s2, REG_ITMP2);
2632                         M_OR(REG_ITMP1, REG_ITMP2);
2633                         emit_bne(cd, iptr->dst.block);
2634                         break;
2635
2636                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2637
2638                         s1 = emit_load_s1_high(jd, iptr, REG_ITMP1);
2639                         s2 = emit_load_s2_high(jd, iptr, REG_ITMP2);
2640                         M_CMP(s2, s1);
2641                         emit_blt(cd, iptr->dst.block);
2642                         s1 = emit_load_s1_low(jd, iptr, REG_ITMP1);
2643                         s2 = emit_load_s2_low(jd, iptr, REG_ITMP2);
2644                         M_BGT(2 + 6);
2645                         M_CMP(s2, s1);
2646                         emit_bult(cd, iptr->dst.block);
2647                         break;
2648
2649                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2650
2651                         s1 = emit_load_s1_high(jd, iptr, REG_ITMP1);
2652                         s2 = emit_load_s2_high(jd, iptr, REG_ITMP2);
2653                         M_CMP(s2, s1);
2654                         emit_bgt(cd, iptr->dst.block);
2655                         s1 = emit_load_s1_low(jd, iptr, REG_ITMP1);
2656                         s2 = emit_load_s2_low(jd, iptr, REG_ITMP2);
2657                         M_BLT(2 + 6);
2658                         M_CMP(s2, s1);
2659                         emit_bugt(cd, iptr->dst.block);
2660                         break;
2661
2662                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2663
2664                         s1 = emit_load_s1_high(jd, iptr, REG_ITMP1);
2665                         s2 = emit_load_s2_high(jd, iptr, REG_ITMP2);
2666                         M_CMP(s2, s1);
2667                         emit_blt(cd, iptr->dst.block);
2668                         s1 = emit_load_s1_low(jd, iptr, REG_ITMP1);
2669                         s2 = emit_load_s2_low(jd, iptr, REG_ITMP2);
2670                         M_BGT(2 + 6);
2671                         M_CMP(s2, s1);
2672                         emit_bule(cd, iptr->dst.block);
2673                         break;
2674
2675                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2676
2677                         s1 = emit_load_s1_high(jd, iptr, REG_ITMP1);
2678                         s2 = emit_load_s2_high(jd, iptr, REG_ITMP2);
2679                         M_CMP(s2, s1);
2680                         emit_bgt(cd, iptr->dst.block);
2681                         s1 = emit_load_s1_low(jd, iptr, REG_ITMP1);
2682                         s2 = emit_load_s2_low(jd, iptr, REG_ITMP2);
2683                         M_BLT(2 + 6);
2684                         M_CMP(s2, s1);
2685                         emit_buge(cd, iptr->dst.block);
2686                         break;
2687
2688
2689                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2690
2691                         REPLACEMENT_POINT_RETURN(cd, iptr);
2692                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2693                         M_INTMOVE(s1, REG_RESULT);
2694                         goto nowperformreturn;
2695
2696                 case ICMD_LRETURN:      /* ..., retvalue ==> ...                      */
2697
2698                         REPLACEMENT_POINT_RETURN(cd, iptr);
2699                         s1 = emit_load_s1(jd, iptr, REG_RESULT_PACKED);
2700                         M_LNGMOVE(s1, REG_RESULT_PACKED);
2701                         goto nowperformreturn;
2702
2703                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2704
2705                         REPLACEMENT_POINT_RETURN(cd, iptr);
2706                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2707                         M_INTMOVE(s1, REG_RESULT);
2708
2709 #ifdef ENABLE_VERIFIER
2710                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2711                                 patcher_add_patch_ref(jd, PATCHER_resolve_class,
2712                                                                         iptr->sx.s23.s2.uc, 0);
2713                         }
2714 #endif /* ENABLE_VERIFIER */
2715                         goto nowperformreturn;
2716
2717                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2718                 case ICMD_DRETURN:
2719
2720                         REPLACEMENT_POINT_RETURN(cd, iptr);
2721                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2722                         goto nowperformreturn;
2723
2724                 case ICMD_RETURN:      /* ...  ==> ...                                */
2725
2726                         REPLACEMENT_POINT_RETURN(cd, iptr);
2727
2728 nowperformreturn:
2729                         {
2730                         s4 i, p;
2731
2732                         p = cd->stackframesize;
2733
2734 #if !defined(NDEBUG)
2735                         emit_verbosecall_exit(jd);
2736 #endif
2737
2738 #if defined(ENABLE_THREADS)
2739                         if (checksync && code_is_synchronized(code)) {
2740                                 M_ALD(REG_ITMP2, REG_SP, rd->memuse * 8);
2741
2742                                 /* we need to save the proper return value */
2743                                 switch (iptr->opc) {
2744                                 case ICMD_IRETURN:
2745                                 case ICMD_ARETURN:
2746                                         M_IST(REG_RESULT, REG_SP, rd->memuse * 8);
2747                                         break;
2748
2749                                 case ICMD_LRETURN:
2750                                         M_LST(REG_RESULT_PACKED, REG_SP, rd->memuse * 8);
2751                                         break;
2752
2753                                 case ICMD_FRETURN:
2754                                         emit_fstps_membase(cd, REG_SP, rd->memuse * 8);
2755                                         break;
2756
2757                                 case ICMD_DRETURN:
2758                                         emit_fstpl_membase(cd, REG_SP, rd->memuse * 8);
2759                                         break;
2760                                 }
2761
2762                                 M_AST(REG_ITMP2, REG_SP, 0);
2763                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP3);
2764                                 M_CALL(REG_ITMP3);
2765
2766                                 /* and now restore the proper return value */
2767                                 switch (iptr->opc) {
2768                                 case ICMD_IRETURN:
2769                                 case ICMD_ARETURN:
2770                                         M_ILD(REG_RESULT, REG_SP, rd->memuse * 8);
2771                                         break;
2772
2773                                 case ICMD_LRETURN:
2774                                         M_LLD(REG_RESULT_PACKED, REG_SP, rd->memuse * 8);
2775                                         break;
2776
2777                                 case ICMD_FRETURN:
2778                                         emit_flds_membase(cd, REG_SP, rd->memuse * 8);
2779                                         break;
2780
2781                                 case ICMD_DRETURN:
2782                                         emit_fldl_membase(cd, REG_SP, rd->memuse * 8);
2783                                         break;
2784                                 }
2785                         }
2786 #endif
2787
2788                         /* restore saved registers */
2789
2790                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2791                                 p--; M_ALD(rd->savintregs[i], REG_SP, p * 8);
2792                         }
2793
2794                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2795                                 p--;
2796                                 emit_fldl_membase(cd, REG_SP, p * 8);
2797                                 if (iptr->opc == ICMD_FRETURN || iptr->opc == ICMD_DRETURN) {
2798                                         assert(0);
2799 /*                                      emit_fstp_reg(cd, rd->savfltregs[i] + fpu_st_offset + 1); */
2800                                 } else {
2801                                         assert(0);
2802 /*                                      emit_fstp_reg(cd, rd->savfltregs[i] + fpu_st_offset); */
2803                                 }
2804                         }
2805
2806                         /* deallocate stack */
2807
2808                         if (cd->stackframesize)
2809                                 M_AADD_IMM(cd->stackframesize * 8 + 4, REG_SP);
2810
2811                         M_RET;
2812                         }
2813                         break;
2814
2815
2816                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2817                         {
2818                                 s4 i, l;
2819                                 branch_target_t *table;
2820
2821                                 table = iptr->dst.table;
2822
2823                                 l = iptr->sx.s23.s2.tablelow;
2824                                 i = iptr->sx.s23.s3.tablehigh;
2825
2826                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2827                                 M_INTMOVE(s1, REG_ITMP1);
2828
2829                                 if (l != 0)
2830                                         M_ISUB_IMM(l, REG_ITMP1);
2831
2832                                 i = i - l + 1;
2833
2834                 /* range check */
2835
2836                                 M_CMP_IMM(i - 1, REG_ITMP1);
2837                                 emit_bugt(cd, table[0].block);
2838
2839                                 /* build jump table top down and use address of lowest entry */
2840
2841                                 table += i;
2842
2843                                 while (--i >= 0) {
2844                                         dseg_add_target(cd, table->block);
2845                                         --table;
2846                                 }
2847
2848                                 /* length of dataseg after last dseg_addtarget is used
2849                                    by load */
2850
2851                                 M_MOV_IMM(0, REG_ITMP2);
2852                                 dseg_adddata(cd);
2853                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 2, REG_ITMP1);
2854                                 M_JMP(REG_ITMP1);
2855                         }
2856                         break;
2857
2858
2859                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2860                         {
2861                                 s4 i;
2862                                 lookup_target_t *lookup;
2863
2864                                 lookup = iptr->dst.lookup;
2865
2866                                 i = iptr->sx.s23.s2.lookupcount;
2867
2868                                 MCODECHECK((i<<2)+8);
2869                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2870
2871                                 while (--i >= 0) {
2872                                         M_CMP_IMM(lookup->value, s1);
2873                                         emit_beq(cd, lookup->target.block);
2874                                         lookup++;
2875                                 }
2876
2877                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2878                                 ALIGNCODENOP;
2879                         }
2880                         break;
2881
2882                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2883
2884                         REPLACEMENT_POINT_FORGC_BUILTIN(cd, iptr);
2885
2886                         bte = iptr->sx.s23.s3.bte;
2887                         md = bte->md;
2888                         goto gen_method;
2889
2890                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2891
2892                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2893                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2894                 case ICMD_INVOKEINTERFACE:
2895
2896                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2897
2898                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2899                                 md = iptr->sx.s23.s3.um->methodref->parseddesc.md;
2900                                 lm = NULL;
2901                         }
2902                         else {
2903                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2904                                 md = lm->parseddesc;
2905                         }
2906
2907 gen_method:
2908                         s3 = md->paramcount;
2909
2910                         MCODECHECK((s3 << 1) + 64);
2911
2912                         /* copy arguments to registers or stack location                  */
2913
2914                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2915                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2916
2917                                 /* Already Preallocated (ARGVAR) ? */
2918                                 if (var->flags & PREALLOC)
2919                                         continue;
2920                                 if (IS_INT_LNG_TYPE(var->type)) {
2921                                         if (!md->params[s3].inmemory) {
2922                                                 log_text("No integer argument registers available!");
2923                                                 assert(0);
2924
2925                                         } else {
2926                                                 if (IS_2_WORD_TYPE(var->type)) {
2927                                                         d = emit_load(jd, iptr, var, REG_ITMP12_PACKED);
2928                                                         M_LST(d, REG_SP, md->params[s3].regoff);
2929                                                 } else {
2930                                                         d = emit_load(jd, iptr, var, REG_ITMP1);
2931                                                         M_IST(d, REG_SP, md->params[s3].regoff);
2932                                                 }
2933                                         }
2934
2935                                 } else {
2936                                         if (!md->params[s3].inmemory) {
2937                                                 s1 = md->params[s3].regoff;
2938                                                 d = emit_load(jd, iptr, var, s1);
2939                                                 M_FLTMOVE(d, s1);
2940
2941                                         } else {
2942                                                 d = emit_load(jd, iptr, var, REG_FTMP1);
2943                                                 if (IS_2_WORD_TYPE(var->type))
2944                                                         M_DST(d, REG_SP, md->params[s3].regoff);
2945                                                 else
2946                                                         M_FST(d, REG_SP, md->params[s3].regoff);
2947                                         }
2948                                 }
2949                         } /* end of for */
2950
2951                         switch (iptr->opc) {
2952                         case ICMD_BUILTIN:
2953                                 d = md->returntype.type;
2954
2955                                 if (bte->stub == NULL) {
2956                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2957                                 }
2958                                 else {
2959                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2960                                 }
2961                                 M_CALL(REG_ITMP1);
2962                                 break;
2963
2964                         case ICMD_INVOKESPECIAL:
2965                                 M_ALD(REG_ITMP1, REG_SP, 0 * 8);
2966                                 emit_nullpointer_check(cd, iptr, REG_ITMP1);
2967                                 /* fall through */
2968
2969                         case ICMD_INVOKESTATIC:
2970                                 if (lm == NULL) {
2971                                         unresolved_method *um = iptr->sx.s23.s3.um;
2972
2973                                         patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
2974                                                                                 um, 0);
2975
2976                                         disp = 0;
2977                                         d = md->returntype.type;
2978                                 }
2979                                 else {
2980                                         disp = (ptrint) lm->stubroutine;
2981                                         d = lm->parseddesc->returntype.type;
2982                                 }
2983
2984                                 M_MOV_IMM(disp, REG_ITMP2);
2985                                 M_CALL(REG_ITMP2);
2986                                 break;
2987
2988                         case ICMD_INVOKEVIRTUAL:
2989                                 M_ALD(REG_ITMP1, REG_SP, 0 * 8);
2990                                 emit_nullpointer_check(cd, iptr, s1);
2991
2992                                 if (lm == NULL) {
2993                                         unresolved_method *um = iptr->sx.s23.s3.um;
2994
2995                                         patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
2996
2997                                         s1 = 0;
2998                                         d = md->returntype.type;
2999                                 }
3000                                 else {
3001                                         s1 = OFFSET(vftbl_t, table[0]) +
3002                                                 sizeof(methodptr) * lm->vftblindex;
3003                                         d = md->returntype.type;
3004                                 }
3005
3006                                 M_ALD(REG_METHODPTR, REG_ITMP1,
3007                                           OFFSET(java_object_t, vftbl));
3008                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
3009                                 M_CALL(REG_ITMP3);
3010                                 break;
3011
3012                         case ICMD_INVOKEINTERFACE:
3013                                 M_ALD(REG_ITMP1, REG_SP, 0 * 8);
3014                                 emit_nullpointer_check(cd, iptr, s1);
3015
3016                                 if (lm == NULL) {
3017                                         unresolved_method *um = iptr->sx.s23.s3.um;
3018
3019                                         patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
3020
3021                                         s1 = 0;
3022                                         s2 = 0;
3023                                         d = md->returntype.type;
3024                                 }
3025                                 else {
3026                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3027                                                 sizeof(methodptr) * lm->class->index;
3028
3029                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3030
3031                                         d = md->returntype.type;
3032                                 }
3033
3034                                 M_ALD(REG_METHODPTR, REG_ITMP1,
3035                                           OFFSET(java_object_t, vftbl));
3036                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
3037                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
3038                                 M_CALL(REG_ITMP3);
3039                                 break;
3040                         }
3041
3042                         /* store size of call code in replacement point */
3043
3044                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
3045                         REPLACEMENT_POINT_FORGC_BUILTIN_RETURN(cd, iptr);
3046
3047                         /* d contains return type */
3048
3049                         if (d != TYPE_VOID) {
3050 #if defined(ENABLE_SSA)
3051                                 if ((ls == NULL) /* || (!IS_TEMPVAR_INDEX(iptr->dst.varindex)) */ ||
3052                                         (ls->lifetime[iptr->dst.varindex].type != UNUSED))
3053                                         /* a "living" stackslot */
3054 #endif
3055                                 {
3056                                         if (IS_INT_LNG_TYPE(d)) {
3057                                                 if (IS_2_WORD_TYPE(d)) {
3058                                                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT_PACKED);
3059                                                         M_LNGMOVE(REG_RESULT_PACKED, s1);
3060                                                 }
3061                                                 else {
3062                                                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
3063                                                         M_INTMOVE(REG_RESULT, s1);
3064                                                 }
3065                                         }
3066                                         else {
3067                                                 s1 = codegen_reg_of_dst(jd, iptr, REG_NULL);
3068                                         }
3069                                         emit_store_dst(jd, iptr, s1);
3070                                 }
3071                         }
3072                         break;
3073
3074
3075                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3076
3077                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
3078                                 /* object type cast-check */
3079
3080                                 classinfo *super;
3081                                 vftbl_t   *supervftbl;
3082                                 s4         superindex;
3083
3084                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3085                                         super = NULL;
3086                                         superindex = 0;
3087                                         supervftbl = NULL;
3088                                 }
3089                                 else {
3090                                         super = iptr->sx.s23.s3.c.cls;
3091                                         superindex = super->index;
3092                                         supervftbl = super->vftbl;
3093                                 }
3094
3095                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE))
3096                                         CODEGEN_CRITICAL_SECTION_NEW;
3097
3098                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
3099
3100                                 /* if class is not resolved, check which code to call */
3101
3102                                 if (super == NULL) {
3103                                         M_TEST(s1);
3104                                         emit_label_beq(cd, BRANCH_LABEL_1);
3105
3106                                         patcher_add_patch_ref(jd, PATCHER_checkcast_instanceof_flags,
3107                                                                                 iptr->sx.s23.s3.c.ref, 0);
3108
3109                                         M_MOV_IMM(0, REG_ITMP2);                  /* super->flags */
3110                                         M_AND_IMM32(ACC_INTERFACE, REG_ITMP2);
3111                                         emit_label_beq(cd, BRANCH_LABEL_2);
3112                                 }
3113
3114                                 /* interface checkcast code */
3115
3116                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
3117                                         if (super != NULL) {
3118                                                 M_TEST(s1);
3119                                                 emit_label_beq(cd, BRANCH_LABEL_3);
3120                                         }
3121
3122                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
3123
3124                                         if (super == NULL) {
3125                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
3126                                                                                         iptr->sx.s23.s3.c.ref,
3127                                                                                         0);
3128                                         }
3129
3130                                         M_ILD32(REG_ITMP3,
3131                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
3132                                         M_ISUB_IMM32(superindex, REG_ITMP3);
3133                                         /* XXX do we need this one? */
3134                                         M_TEST(REG_ITMP3);
3135                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
3136
3137                                         M_ALD32(REG_ITMP3, REG_ITMP2,
3138                                                         OFFSET(vftbl_t, interfacetable[0]) -
3139                                                         superindex * sizeof(methodptr*));
3140                                         M_TEST(REG_ITMP3);
3141                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
3142
3143                                         if (super == NULL)
3144                                                 emit_label_br(cd, BRANCH_LABEL_4);
3145                                         else
3146                                                 emit_label(cd, BRANCH_LABEL_3);
3147                                 }
3148
3149                                 /* class checkcast code */
3150
3151                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
3152                                         if (super == NULL) {
3153                                                 emit_label(cd, BRANCH_LABEL_2);
3154                                         }
3155                                         else {
3156                                                 M_TEST(s1);
3157                                                 emit_label_beq(cd, BRANCH_LABEL_5);
3158                                         }
3159
3160                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
3161
3162                                         if (super == NULL) {
3163                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_class,
3164                                                                                         iptr->sx.s23.s3.c.ref,
3165                                                                                         0);
3166                                         }
3167
3168                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3169
3170                                         CODEGEN_CRITICAL_SECTION_START;
3171
3172                                         M_ILD32(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
3173
3174                                         /*                              if (s1 != REG_ITMP1) { */
3175                                         /*                                      emit_mov_membase_reg(cd, REG_ITMP3, OFFSET(vftbl_t, baseval), REG_ITMP1); */
3176                                         /*                                      emit_mov_membase_reg(cd, REG_ITMP3, OFFSET(vftbl_t, diffval), REG_ITMP3); */
3177                                         /* #if defined(ENABLE_THREADS) */
3178                                         /*                                      codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3179                                         /* #endif */
3180                                         /*                                      emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
3181
3182                                         /*                              } else { */
3183                                         M_ILD32(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
3184                                         M_ISUB(REG_ITMP3, REG_ITMP2);
3185                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3186                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3187
3188                                         CODEGEN_CRITICAL_SECTION_END;
3189
3190                                         /*                              } */
3191
3192                                         M_CMP(REG_ITMP3, REG_ITMP2);
3193                                         emit_classcast_check(cd, iptr, BRANCH_ULE, REG_ITMP3, s1);
3194
3195                                         if (super != NULL)
3196                                                 emit_label(cd, BRANCH_LABEL_5);
3197                                 }
3198
3199                                 if (super == NULL) {
3200                                         emit_label(cd, BRANCH_LABEL_1);
3201                                         emit_label(cd, BRANCH_LABEL_4);
3202                                 }
3203
3204                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
3205                         }
3206                         else {
3207                                 /* array type cast-check */
3208
3209                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
3210                                 M_AST(s1, REG_SP, 0 * 4);
3211
3212                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3213                                         patcher_add_patch_ref(jd, PATCHER_builtin_arraycheckcast,
3214                                                                                 iptr->sx.s23.s3.c.ref, 0);
3215                                 }
3216
3217                                 M_AST_IMM(iptr->sx.s23.s3.c.cls, REG_SP, 1 * 4);
3218                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP3);
3219                                 M_CALL(REG_ITMP3);
3220
3221                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
3222                                 M_TEST(REG_RESULT);
3223                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
3224
3225                                 d = codegen_reg_of_dst(jd, iptr, s1);
3226                         }
3227
3228                         M_INTMOVE(s1, d);
3229                         emit_store_dst(jd, iptr, d);
3230                         break;
3231
3232                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3233
3234                         {
3235                         classinfo *super;
3236                         vftbl_t   *supervftbl;
3237                         s4         superindex;
3238
3239                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3240                                 super = NULL;
3241                                 superindex = 0;
3242                                 supervftbl = NULL;
3243
3244                         } else {
3245                                 super = iptr->sx.s23.s3.c.cls;
3246                                 superindex = super->index;
3247                                 supervftbl = super->vftbl;
3248                         }
3249
3250                         if ((super == NULL) || !(super->flags & ACC_INTERFACE))
3251                                 CODEGEN_CRITICAL_SECTION_NEW;
3252
3253                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
3254                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
3255
3256                         if (s1 == d) {
3257                                 M_INTMOVE(s1, REG_ITMP1);
3258                                 s1 = REG_ITMP1;
3259                         }
3260
3261                         M_CLR(d);
3262
3263                         /* if class is not resolved, check which code to call */
3264
3265                         if (super == NULL) {
3266                                 M_TEST(s1);
3267                                 emit_label_beq(cd, BRANCH_LABEL_1);
3268
3269                                 patcher_add_patch_ref(jd, PATCHER_checkcast_instanceof_flags,
3270                                                                         iptr->sx.s23.s3.c.ref, 0);
3271
3272                                 M_MOV_IMM(0, REG_ITMP3);                      /* super->flags */
3273                                 M_AND_IMM32(ACC_INTERFACE, REG_ITMP3);
3274                                 emit_label_beq(cd, BRANCH_LABEL_2);
3275                         }
3276
3277                         /* interface instanceof code */
3278
3279                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
3280                                 if (super != NULL) {
3281                                         M_TEST(s1);
3282                                         emit_label_beq(cd, BRANCH_LABEL_3);
3283                                 }
3284
3285                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
3286
3287                                 if (super == NULL) {
3288                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
3289                                                                                 iptr->sx.s23.s3.c.ref, 0);
3290                                 }
3291
3292                                 M_ILD32(REG_ITMP3,
3293                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
3294                                 M_ISUB_IMM32(superindex, REG_ITMP3);
3295                                 M_TEST(REG_ITMP3);
3296
3297                                 disp = (2 + 4 /* mov_membase32_reg */ + 2 /* test */ +
3298                                                 6 /* jcc */ + 5 /* mov_imm_reg */);
3299
3300                                 M_BLE(disp);
3301                                 M_ALD32(REG_ITMP1, REG_ITMP1,
3302                                                 OFFSET(vftbl_t, interfacetable[0]) -
3303                                                 superindex * sizeof(methodptr*));
3304                                 M_TEST(REG_ITMP1);
3305 /*                                      emit_setcc_reg(cd, CC_A, d); */
3306 /*                                      emit_jcc(cd, CC_BE, 5); */
3307                                 M_BEQ(5);
3308                                 M_MOV_IMM(1, d);
3309
3310                                 if (super == NULL)
3311                                         emit_label_br(cd, BRANCH_LABEL_4);
3312                                 else
3313                                         emit_label(cd, BRANCH_LABEL_3);
3314                         }
3315
3316                         /* class instanceof code */
3317
3318                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
3319                                 if (super == NULL) {
3320                                         emit_label(cd, BRANCH_LABEL_2);
3321                                 }
3322                                 else {
3323                                         M_TEST(s1);
3324                                         emit_label_beq(cd, BRANCH_LABEL_5);
3325                                 }
3326
3327                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
3328
3329                                 if (super == NULL) {
3330                                         patcher_add_patch_ref(jd, PATCHER_instanceof_class,
3331                                                                                 iptr->sx.s23.s3.c.ref, 0);
3332                                 }
3333
3334                                 M_MOV_IMM(supervftbl, REG_ITMP2);
3335
3336                                 CODEGEN_CRITICAL_SECTION_START;
3337
3338                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
3339                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3340                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
3341
3342                                 CODEGEN_CRITICAL_SECTION_END;
3343
3344                                 M_ISUB(REG_ITMP2, REG_ITMP1);
3345                                 M_CLR(d);                                 /* may be REG_ITMP2 */
3346                                 M_CMP(REG_ITMP3, REG_ITMP1);
3347                                 M_BA(5);
3348                                 M_MOV_IMM(1, d);
3349
3350                                 if (super != NULL)
3351                                         emit_label(cd, BRANCH_LABEL_5);
3352                         }
3353
3354                         if (super == NULL) {
3355                                 emit_label(cd, BRANCH_LABEL_1);
3356                                 emit_label(cd, BRANCH_LABEL_4);
3357                         }
3358
3359                         emit_store_dst(jd, iptr, d);
3360                         }
3361                         break;
3362
3363                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3364
3365                         /* check for negative sizes and copy sizes to stack if necessary  */
3366
3367                         MCODECHECK((iptr->s1.argcount << 1) + 64);
3368
3369                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
3370                                 /* copy SAVEDVAR sizes to stack */
3371                                 var = VAR(iptr->sx.s23.s2.args[s1]);
3372
3373                                 /* Already Preallocated? */
3374                                 if (!(var->flags & PREALLOC)) {
3375                                         if (var->flags & INMEMORY) {
3376                                                 M_ILD(REG_ITMP1, REG_SP, var->vv.regoff);
3377                                                 M_IST(REG_ITMP1, REG_SP, (s1 + 3) * 4);
3378                                         }
3379                                         else
3380                                                 M_IST(var->vv.regoff, REG_SP, (s1 + 3) * 4);
3381                                 }
3382                         }
3383
3384                         /* is a patcher function set? */
3385
3386                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3387                                 patcher_add_patch_ref(jd, PATCHER_builtin_multianewarray,
3388                                                                         iptr->sx.s23.s3.c.ref, 0);
3389
3390                                 disp = 0;
3391
3392                         }
3393                         else
3394                                 disp = (ptrint) iptr->sx.s23.s3.c.cls;
3395
3396                         /* a0 = dimension count */
3397
3398                         M_IST_IMM(iptr->s1.argcount, REG_SP, 0 * 4);
3399
3400                         /* a1 = arraydescriptor */
3401
3402                         M_IST_IMM(disp, REG_SP, 1 * 4);
3403
3404                         /* a2 = pointer to dimensions = stack pointer */
3405
3406                         M_MOV(REG_SP, REG_ITMP1);
3407                         M_AADD_IMM(3 * 4, REG_ITMP1);
3408                         M_AST(REG_ITMP1, REG_SP, 2 * 4);
3409
3410                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3411                         M_CALL(REG_ITMP1);
3412
3413                         /* check for exception before result assignment */
3414
3415                         emit_exception_check(cd, iptr);
3416
3417                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
3418                         M_INTMOVE(REG_RESULT, s1);
3419                         emit_store_dst(jd, iptr, s1);
3420                         break;
3421
3422                 default:
3423                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
3424                                                                                    iptr->opc);
3425                         return false;
3426         } /* switch */
3427
3428         } /* for instruction */
3429
3430         MCODECHECK(64);
3431
3432 #if defined(ENABLE_LSRA) && !defined(ENABLE_SSA)
3433         if (!opt_lsra)
3434 #endif
3435 #if defined(ENABLE_SSA)
3436         if ( ls != NULL ) {
3437
3438                 /* by edge splitting, in Blocks with phi moves there can only */
3439                 /* be a goto as last command, no other Jump/Branch Command    */
3440
3441                 if (!last_cmd_was_goto)
3442                         codegen_emit_phi_moves(jd, bptr);
3443         }
3444
3445 #endif
3446
3447         /* At the end of a basic block we may have to append some nops,
3448            because the patcher stub calling code might be longer than the
3449            actual instruction. So codepatching does not change the
3450            following block unintentionally. */
3451
3452         if (cd->mcodeptr < cd->lastmcodeptr) {
3453                 while (cd->mcodeptr < cd->lastmcodeptr) {
3454                         M_NOP;
3455                 }
3456         }
3457
3458         } /* if (bptr -> flags >= BBREACHED) */
3459         } /* for basic block */
3460
3461         /* generate stubs */
3462
3463         emit_patcher_traps(jd);
3464
3465         /* everything's ok */
3466
3467         return true;
3468 }
3469
3470
3471 /* codegen_emit_stub_native ****************************************************
3472
3473    Emits a stub routine which calls a native method.
3474
3475 *******************************************************************************/
3476
3477 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f, int skipparams)
3478 {
3479         methodinfo  *m;
3480         codeinfo    *code;
3481         codegendata *cd;
3482         methoddesc  *md;
3483         int          i, j;                 /* count variables                    */
3484         int          s1, s2;
3485         int          disp;
3486
3487         /* get required compiler data */
3488
3489         m    = jd->m;
3490         code = jd->code;
3491         cd   = jd->cd;
3492
3493         /* set some variables */
3494
3495         md = m->parseddesc;
3496
3497         /* calculate stackframe size */
3498
3499         cd->stackframesize =
3500                 sizeof(stackframeinfo_t) / SIZEOF_VOID_P +
3501                 sizeof(localref_table) / SIZEOF_VOID_P +
3502                 4 +                             /* 4 arguments (start_native_call)    */
3503                 nmd->memuse;
3504
3505     /* keep stack 16-byte aligned */
3506
3507         ALIGN_ODD(cd->stackframesize);
3508
3509         /* create method header */
3510
3511         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
3512         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8 + 4); /* FrameSize       */
3513         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
3514         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
3515         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
3516
3517 #if defined(ENABLE_PROFILING)
3518         /* generate native method profiling code */
3519
3520         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3521                 /* count frequency */
3522
3523                 M_MOV_IMM(code, REG_ITMP1);
3524                 M_IADD_IMM_MEMBASE(1, REG_ITMP1, OFFSET(codeinfo, frequency));
3525         }
3526 #endif
3527
3528         /* calculate stackframe size for native function */
3529
3530         M_ASUB_IMM(cd->stackframesize * 8 + 4, REG_SP);
3531
3532         /* Mark the whole fpu stack as free for native functions (only for saved  */
3533         /* register count == 0).                                                  */
3534
3535         emit_ffree_reg(cd, 0);
3536         emit_ffree_reg(cd, 1);
3537         emit_ffree_reg(cd, 2);
3538         emit_ffree_reg(cd, 3);
3539         emit_ffree_reg(cd, 4);
3540         emit_ffree_reg(cd, 5);
3541         emit_ffree_reg(cd, 6);
3542         emit_ffree_reg(cd, 7);
3543
3544 #if defined(ENABLE_GC_CACAO)
3545         /* remember callee saved int registers in stackframeinfo (GC may need to  */
3546         /* recover them during a collection).                                     */
3547
3548         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
3549                 OFFSET(stackframeinfo_t, intregs);
3550
3551         for (i = 0; i < INT_SAV_CNT; i++)
3552                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 4);
3553 #endif
3554
3555         /* prepare data structures for native function call */
3556
3557         M_MOV(REG_SP, REG_ITMP1);
3558         M_AST(REG_ITMP1, REG_SP, 0 * 4);
3559         M_IST_IMM(0, REG_SP, 1 * 4);
3560         dseg_adddata(cd);
3561
3562         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3563         M_CALL(REG_ITMP1);
3564
3565         /* remember class argument */
3566
3567         if (m->flags & ACC_STATIC)
3568                 M_MOV(REG_RESULT, REG_ITMP3);
3569
3570         /* Copy or spill arguments to new locations. */
3571
3572         for (i = md->paramcount - 1, j = i + skipparams; i >= 0; i--, j--) {
3573                 if (!md->params[i].inmemory)
3574                         assert(0);
3575
3576                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;
3577                 s2 = nmd->params[j].regoff;
3578
3579                 /* float/double in memory can be copied like int/longs */
3580
3581                 switch (md->paramtypes[i].type) {
3582                 case TYPE_INT:
3583                 case TYPE_FLT:
3584                 case TYPE_ADR:
3585                         M_ILD(REG_ITMP1, REG_SP, s1);
3586                         M_IST(REG_ITMP1, REG_SP, s2);
3587                         break;
3588                 case TYPE_LNG:
3589                 case TYPE_DBL:
3590                         M_LLD(REG_ITMP12_PACKED, REG_SP, s1);
3591                         M_LST(REG_ITMP12_PACKED, REG_SP, s2);
3592                         break;
3593                 }
3594         }
3595
3596         /* Handle native Java methods. */
3597
3598         if (m->flags & ACC_NATIVE) {
3599                 /* if function is static, put class into second argument */
3600
3601                 if (m->flags & ACC_STATIC)
3602                         M_AST(REG_ITMP3, REG_SP, 1 * 4);
3603
3604                 /* put env into first argument */
3605
3606                 M_AST_IMM(_Jv_env, REG_SP, 0 * 4);
3607         }
3608
3609         /* Call the native function. */
3610
3611         disp = dseg_add_functionptr(cd, f);
3612         emit_mov_imm_reg(cd, 0, REG_ITMP3);
3613         dseg_adddata(cd);
3614         M_ALD(REG_ITMP1, REG_ITMP3, disp);
3615         M_CALL(REG_ITMP1);
3616
3617         /* save return value */
3618
3619         switch (md->returntype.type) {
3620         case TYPE_INT:
3621         case TYPE_ADR:
3622                 M_IST(REG_RESULT, REG_SP, 1 * 8);
3623                 break;
3624         case TYPE_LNG:
3625                 M_LST(REG_RESULT_PACKED, REG_SP, 1 * 8);
3626                 break;
3627         case TYPE_FLT:
3628                 emit_fsts_membase(cd, REG_SP, 1 * 8);
3629                 break;
3630         case TYPE_DBL:
3631                 emit_fstl_membase(cd, REG_SP, 1 * 8);
3632                 break;
3633         case TYPE_VOID:
3634                 break;
3635         }
3636
3637         /* remove native stackframe info */
3638
3639         M_MOV(REG_SP, REG_ITMP1);
3640         M_AST(REG_ITMP1, REG_SP, 0 * 4);
3641         M_IST_IMM(0, REG_SP, 1 * 4);
3642         dseg_adddata(cd);
3643
3644         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3645         M_CALL(REG_ITMP1);
3646         M_MOV(REG_RESULT, REG_ITMP2);                 /* REG_ITMP3 == REG_RESULT2 */
3647
3648         /* restore return value */
3649
3650         switch (md->returntype.type) {
3651         case TYPE_INT:
3652         case TYPE_ADR:
3653                 M_ILD(REG_RESULT, REG_SP, 1 * 8);
3654                 break;
3655         case TYPE_LNG:
3656                 M_LLD(REG_RESULT_PACKED, REG_SP, 1 * 8);
3657                 break;
3658         case TYPE_FLT:
3659                 emit_flds_membase(cd, REG_SP, 1 * 8);
3660                 break;
3661         case TYPE_DBL:
3662                 emit_fldl_membase(cd, REG_SP, 1 * 8);
3663                 break;
3664         case TYPE_VOID:
3665                 break;
3666         }
3667
3668 #if defined(ENABLE_GC_CACAO)
3669         /* restore callee saved int registers from stackframeinfo (GC might have  */
3670         /* modified them during a collection).                                    */
3671
3672         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
3673                 OFFSET(stackframeinfo_t, intregs);
3674
3675         for (i = 0; i < INT_SAV_CNT; i++)
3676                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 4);
3677 #endif
3678
3679         M_AADD_IMM(cd->stackframesize * 8 + 4, REG_SP);
3680
3681         /* check for exception */
3682
3683         M_TEST(REG_ITMP2);
3684         M_BNE(1);
3685
3686         M_RET;
3687
3688         /* handle exception */
3689
3690         M_MOV(REG_ITMP2, REG_ITMP1_XPTR);
3691         M_ALD(REG_ITMP2_XPC, REG_SP, 0);
3692         M_ASUB_IMM(2, REG_ITMP2_XPC);
3693
3694         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3695         M_JMP(REG_ITMP3);
3696 }
3697
3698
3699 /*
3700  * These are local overrides for various environment variables in Emacs.
3701  * Please do not remove this and leave it at the end of the file, where
3702  * Emacs will automagically detect them.
3703  * ---------------------------------------------------------------------
3704  * Local variables:
3705  * mode: c
3706  * indent-tabs-mode: t
3707  * c-basic-offset: 4
3708  * tab-width: 4
3709  * End:
3710  * vim:noexpandtab:sw=4:ts=4:
3711  */