src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006, 2007, 2008
   4    CACAOVM - Verein zur Foerderung der freien virtuellen Maschine CACAO
   5
   6    This file is part of CACAO.
   7
   8    This program is free software; you can redistribute it and/or
   9    modify it under the terms of the GNU General Public License as
  10    published by the Free Software Foundation; either version 2, or (at
  11    your option) any later version.
  12
  13    This program is distributed in the hope that it will be useful, but
  14    WITHOUT ANY WARRANTY; without even the implied warranty of
  15    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16    General Public License for more details.
  17
  18    You should have received a copy of the GNU General Public License
  19    along with this program; if not, write to the Free Software
  20    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  21    02110-1301, USA.
  22
  23 */
  24
  25
  26 #include "config.h"
  27
  28 #include <assert.h>
  29 #include <stdio.h>
  30 #include <stdint.h>
  31
  32 #include "vm/types.h"
  33
  34 #include "md-abi.h"
  35
  36 #include "vm/jit/x86_64/arch.h"
  37 #include "vm/jit/x86_64/codegen.h"
  38 #include "vm/jit/x86_64/emit.h"
  39
  40 #include "mm/memory.h"
  41
  42 #include "native/jni.h"
  43 #include "native/localref.h"
  44 #include "native/native.h"
  45
  46 #include "threads/lock-common.h"
  47
  48 #include "vm/builtin.h"
  49 #include "vm/exceptions.hpp"
  50 #include "vm/global.h"
  51 #include "vm/loader.h"
  52 #include "vm/options.h"
  53 #include "vm/primitive.hpp"
  54 #include "vm/statistics.h"
  55 #include "vm/string.hpp"
  56 #include "vm/vm.hpp"
  57
  58 #include "vm/jit/abi.h"
  59 #include "vm/jit/asmpart.h"
  60 #include "vm/jit/code.h"
  61 #include "vm/jit/codegen-common.h"
  62 #include "vm/jit/dseg.h"
  63 #include "vm/jit/emit-common.h"
  64 #include "vm/jit/jit.h"
  65 #include "vm/jit/linenumbertable.h"
  66 #include "vm/jit/methodheader.h"
  67 #include "vm/jit/parse.h"
  68 #include "vm/jit/patcher-common.h"
  69 #include "vm/jit/reg.h"
  70 #include "vm/jit/replace.h"
  71 #include "vm/jit/stacktrace.hpp"
  72 #include "vm/jit/trap.h"
  73
  74 #if defined(ENABLE_LSRA)
  75 # include "vm/jit/allocator/lsra.h"
  76 #endif
  77
  78
  79 /* codegen_emit ****************************************************************
  80
  81    Generates machine code.
  82
  83 *******************************************************************************/
  84
  85 bool codegen_emit(jitdata *jd)
  86 {
  87         methodinfo         *m;
  88         codeinfo           *code;
  89         codegendata        *cd;
  90         registerdata       *rd;
  91         s4                  len, s1, s2, s3, d, disp;
  92         u2                  currentline;
  93         ptrint              a;
  94         varinfo            *var, *dst;
  95         basicblock         *bptr;
  96         instruction        *iptr;
  97         constant_classref  *cr;
  98         unresolved_class   *uc;
  99         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 100         unresolved_method  *um;
 101         builtintable_entry *bte;
 102         methoddesc         *md;
 103         fieldinfo          *fi;
 104         unresolved_field   *uf;
 105         s4                  fieldtype;
 106         s4                 varindex;
 107
 108         /* get required compiler data */
 109
 110         m    = jd->m;
 111         code = jd->code;
 112         cd   = jd->cd;
 113         rd   = jd->rd;
 114
 115         /* prevent compiler warnings */
 116
 117         d   = 0;
 118         lm  = NULL;
 119         um  = NULL;
 120         bte = NULL;
 121
 122         {
 123         s4 i, p, t, l;
 124         s4 savedregs_num;
 125
 126         savedregs_num = 0;
 127
 128         /* space to save used callee saved registers */
 129
 130         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 131         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 132
 133         cd->stackframesize = rd->memuse + savedregs_num;
 134
 135 #if defined(ENABLE_THREADS)
 136         /* space to save argument of monitor_enter */
 137
 138         if (checksync && code_is_synchronized(code))
 139                 cd->stackframesize++;
 140 #endif
 141
 142         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 143            native code e.g. libc or jni (alignment problems with
 144            movaps). */
 145
 146         if (!code_is_leafmethod(code) || opt_verbosecall)
 147                 cd->stackframesize |= 0x1;
 148
 149         /* create method header */
 150
 151         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 152         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 153
 154         code->synchronizedoffset = rd->memuse * 8;
 155
 156         if (code_is_leafmethod(code))
 157                 (void) dseg_add_unique_s4(cd, 1);                  /* IsLeaf          */
 158         else
 159                 (void) dseg_add_unique_s4(cd, 0);                  /* IsLeaf          */
 160
 161         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 162         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 163
 164 #if defined(ENABLE_PROFILING)
 165         /* generate method profiling code */
 166
 167         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 168                 /* count frequency */
 169
 170                 M_MOV_IMM(code, REG_ITMP3);
 171                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 172
 173                 PROFILE_CYCLE_START;
 174         }
 175 #endif
 176
 177         /* create stack frame (if necessary) */
 178
 179         if (cd->stackframesize)
 180                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 181
 182         /* save used callee saved registers */
 183
 184         p = cd->stackframesize;
 185         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 186                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 187         }
 188         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 189                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 190         }
 191
 192         /* take arguments out of register or stack frame */
 193
 194         md = m->parseddesc;
 195
 196         for (p = 0, l = 0; p < md->paramcount; p++) {
 197                 t = md->paramtypes[p].type;
 198
 199                 varindex = jd->local_map[l * 5 + t];
 200
 201                 l++;
 202                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 203                         l++;
 204
 205                 if (varindex == UNUSED)
 206                         continue;
 207
 208                 var = VAR(varindex);
 209
 210                 s1 = md->params[p].regoff;
 211
 212                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 213                         if (!md->params[p].inmemory) {           /* register arguments    */
 214                                 if (!IS_INMEMORY(var->flags))
 215                                         M_INTMOVE(s1, var->vv.regoff);
 216                                 else
 217                                     M_LST(s1, REG_SP, var->vv.regoff);
 218                         }
 219                         else {                                 /* stack arguments       */
 220                                 if (!IS_INMEMORY(var->flags))
 221                                         /* + 8 for return address */
 222                                         M_LLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 223                                 else
 224                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 225                         }
 226                 }
 227                 else {                                     /* floating args         */
 228                         if (!md->params[p].inmemory) {           /* register arguments    */
 229                                 if (!IS_INMEMORY(var->flags))
 230                                         M_FLTMOVE(s1, var->vv.regoff);
 231                                 else
 232                                         M_DST(s1, REG_SP, var->vv.regoff);
 233                         }
 234                         else {                                 /* stack arguments       */
 235                                 if (!IS_INMEMORY(var->flags))
 236                                         M_DLD(var->vv.regoff, REG_SP, cd->stackframesize * 8 + s1 + 8);
 237                                 else
 238                                         var->vv.regoff = cd->stackframesize * 8 + s1 + 8;
 239                         }
 240                 }
 241         }
 242
 243         /* save monitorenter argument */
 244
 245 #if defined(ENABLE_THREADS)
 246         if (checksync && code_is_synchronized(code)) {
 247                 /* stack offset for monitor argument */
 248
 249                 s1 = rd->memuse;
 250
 251                 if (opt_verbosecall) {
 252                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 253
 254                         for (p = 0; p < INT_ARG_CNT; p++)
 255                                 M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
 256
 257                         for (p = 0; p < FLT_ARG_CNT; p++)
 258                                 M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 259
 260                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 261                 }
 262
 263                 /* decide which monitor enter function to call */
 264
 265                 if (m->flags & ACC_STATIC) {
 266                         M_MOV_IMM(&m->clazz->object.header, REG_A0);
 267                 }
 268                 else {
 269                         M_TEST(REG_A0);
 270                         M_BNE(8);
 271                         M_ALD_MEM(REG_A0, TRAP_NullPointerException);
 272                 }
 273
 274                 M_AST(REG_A0, REG_SP, s1 * 8);
 275                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 276                 M_CALL(REG_ITMP1);
 277
 278                 if (opt_verbosecall) {
 279                         for (p = 0; p < INT_ARG_CNT; p++)
 280                                 M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 281
 282                         for (p = 0; p < FLT_ARG_CNT; p++)
 283                                 M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
 284
 285                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 286                 }
 287         }
 288 #endif
 289
 290 #if !defined(NDEBUG)
 291         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 292                 emit_verbosecall_enter(jd);
 293 #endif /* !defined(NDEBUG) */
 294
 295         }
 296
 297         /* end of header generation */
 298
 299         /* create replacement points */
 300
 301         REPLACEMENT_POINTS_INIT(cd, jd);
 302
 303         /* walk through all basic blocks */
 304
 305         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 306
 307                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 308
 309                 if (bptr->flags >= BBREACHED) {
 310
 311                 /* branch resolving */
 312
 313                 codegen_resolve_branchrefs(cd, bptr);
 314
 315                 /* handle replacement points */
 316
 317                 REPLACEMENT_POINT_BLOCK_START(cd, bptr);
 318
 319                 /* copy interface registers to their destination */
 320
 321                 len = bptr->indepth;
 322                 MCODECHECK(512);
 323
 324 #if defined(ENABLE_PROFILING)
 325                 /* generate basicblock profiling code */
 326
 327                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 328                         /* count frequency */
 329
 330                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 331                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 332
 333                         /* if this is an exception handler, start profiling again */
 334
 335                         if (bptr->type == BBTYPE_EXH)
 336                                 PROFILE_CYCLE_START;
 337                 }
 338 #endif
 339
 340 #if defined(ENABLE_LSRA)
 341                 if (opt_lsra) {
 342                         while (len) {
 343                                 len--;
 344                                 src = bptr->invars[len];
 345                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 346                                         if (bptr->type == BBTYPE_EXH) {
 347 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 348                                                 if (!IS_INMEMORY(src->flags))
 349                                                         d= src->vv.regoff;
 350                                                 else
 351                                                         d=REG_ITMP1;
 352                                                 M_INTMOVE(REG_ITMP1, d);
 353                                                 emit_store(jd, NULL, src, d);
 354                                         }
 355                                 }
 356                         }
 357
 358                 } else {
 359 #endif
 360
 361                 while (len) {
 362                         len--;
 363                         var = VAR(bptr->invars[len]);
 364                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 365                                 if (bptr->type == BBTYPE_EXH) {
 366                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 367                                         M_INTMOVE(REG_ITMP1, d);
 368                                         emit_store(jd, NULL, var, d);
 369                                 }
 370                         }
 371                         else {
 372                                 assert((var->flags & INOUT));
 373                         }
 374                 }
 375 #if defined(ENABLE_LSRA)
 376                 }
 377 #endif
 378                 /* walk through all instructions */
 379
 380                 len = bptr->icount;
 381                 currentline = 0;
 382
 383                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 384                         if (iptr->line != currentline) {
 385                                 linenumbertable_list_entry_add(cd, iptr->line);
 386                                 currentline = iptr->line;
 387                         }
 388
 389                         MCODECHECK(1024);                         /* 1KB should be enough */
 390
 391                 switch (iptr->opc) {
 392                 case ICMD_NOP:        /* ...  ==> ...                                 */
 393                 case ICMD_POP:        /* ..., value  ==> ...                          */
 394                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 395                         break;
 396
 397                 case ICMD_INLINE_START:
 398
 399                         REPLACEMENT_POINT_INLINE_START(cd, iptr);
 400                         break;
 401
 402                 case ICMD_INLINE_BODY:
 403
 404                         REPLACEMENT_POINT_INLINE_BODY(cd, iptr);
 405                         linenumbertable_list_entry_add_inline_start(cd, iptr);
 406                         linenumbertable_list_entry_add(cd, iptr->line);
 407                         break;
 408
 409                 case ICMD_INLINE_END:
 410
 411                         linenumbertable_list_entry_add_inline_end(cd, iptr);
 412                         linenumbertable_list_entry_add(cd, iptr->line);
 413                         break;
 414
 415                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 416
 417                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 418                         emit_nullpointer_check(cd, iptr, s1);
 419                         break;
 420
 421                 /* constant operations ************************************************/
 422
 423                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 424
 425                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 426                         ICONST(d, iptr->sx.val.i);
 427                         emit_store_dst(jd, iptr, d);
 428                         break;
 429
 430                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 431
 432                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 433                         LCONST(d, iptr->sx.val.l);
 434                         emit_store_dst(jd, iptr, d);
 435                         break;
 436
 437                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 438
 439                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 440                         disp = dseg_add_float(cd, iptr->sx.val.f);
 441                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 442                         emit_store_dst(jd, iptr, d);
 443                         break;
 444
 445                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 446
 447                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 448                         disp = dseg_add_double(cd, iptr->sx.val.d);
 449                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 450                         emit_store_dst(jd, iptr, d);
 451                         break;
 452
 453                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 454
 455                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 456
 457                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 458                                 cr   = iptr->sx.val.c.ref;
 459                                 disp = dseg_add_unique_address(cd, cr);
 460
 461 /*                              PROFILE_CYCLE_STOP; */
 462
 463                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
 464                                                                           cr, disp);
 465
 466 /*                              PROFILE_CYCLE_START; */
 467
 468                                 M_ALD(d, RIP, disp);
 469                         }
 470                         else {
 471                                 if (iptr->sx.val.anyptr == 0) {
 472                                         M_CLR(d);
 473                                 }
 474                                 else {
 475                                         disp = dseg_add_address(cd, iptr->sx.val.anyptr);
 476                                         M_ALD(d, RIP, disp);
 477                                 }
 478                         }
 479                         emit_store_dst(jd, iptr, d);
 480                         break;
 481
 482
 483                 /* load/store/copy/move operations ************************************/
 484
 485                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 486                 case ICMD_ALOAD:      /* s1 = local variable                          */
 487                 case ICMD_LLOAD:
 488                 case ICMD_FLOAD:
 489                 case ICMD_DLOAD:
 490                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 491                 case ICMD_LSTORE:
 492                 case ICMD_FSTORE:
 493                 case ICMD_DSTORE:
 494                 case ICMD_COPY:
 495                 case ICMD_MOVE:
 496
 497                         emit_copy(jd, iptr);
 498                         break;
 499
 500                 case ICMD_ASTORE:
 501                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 502                                 emit_copy(jd, iptr);
 503                         break;
 504
 505                 /* integer operations *************************************************/
 506
 507                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 508
 509                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 510                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 511                         M_INTMOVE(s1, d);
 512                         M_INEG(d);
 513                         emit_store_dst(jd, iptr, d);
 514                         break;
 515
 516                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 517
 518                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 519                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 520                         M_INTMOVE(s1, d);
 521                         M_LNEG(d);
 522                         emit_store_dst(jd, iptr, d);
 523                         break;
 524
 525                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 526
 527                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 528                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 529                         M_ISEXT(s1, d);
 530                         emit_store_dst(jd, iptr, d);
 531                         break;
 532
 533                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 534
 535                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 536                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 537                         M_IMOV(s1, d);
 538                         emit_store_dst(jd, iptr, d);
 539                         break;
 540
 541                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 542
 543                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 544                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 545                         M_BSEXT(s1, d);
 546                         emit_store_dst(jd, iptr, d);
 547                         break;
 548
 549                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 550
 551                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 552                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 553                         M_CZEXT(s1, d);
 554                         emit_store_dst(jd, iptr, d);
 555                         break;
 556
 557                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 558
 559                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 560                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 561                         M_SSEXT(s1, d);
 562                         emit_store_dst(jd, iptr, d);
 563                         break;
 564
 565
 566                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 567
 568                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 569                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 570                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 571                         if (s2 == d)
 572                                 M_IADD(s1, d);
 573                         else {
 574                                 M_INTMOVE(s1, d);
 575                                 M_IADD(s2, d);
 576                         }
 577                         emit_store_dst(jd, iptr, d);
 578                         break;
 579
 580                 case ICMD_IINC:
 581                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 582                                       /* sx.val.i = constant                             */
 583
 584                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 585                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 586
 587                         /* Using inc and dec is not faster than add (tested with
 588                            sieve). */
 589
 590                         M_INTMOVE(s1, d);
 591                         M_IADD_IMM(iptr->sx.val.i, d);
 592                         emit_store_dst(jd, iptr, d);
 593                         break;
 594
 595                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 596
 597                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 598                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 599                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 600                         if (s2 == d)
 601                                 M_LADD(s1, d);
 602                         else {
 603                                 M_INTMOVE(s1, d);
 604                                 M_LADD(s2, d);
 605                         }
 606                         emit_store_dst(jd, iptr, d);
 607                         break;
 608
 609                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 610                                       /* sx.val.l = constant                             */
 611
 612                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 613                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 614                         M_INTMOVE(s1, d);
 615                         if (IS_IMM32(iptr->sx.val.l))
 616                                 M_LADD_IMM(iptr->sx.val.l, d);
 617                         else {
 618                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 619                                 M_LADD(REG_ITMP2, d);
 620                         }
 621                         emit_store_dst(jd, iptr, d);
 622                         break;
 623
 624                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 625
 626                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 627                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 628                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 629                         if (s2 == d) {
 630                                 M_INTMOVE(s1, REG_ITMP1);
 631                                 M_ISUB(s2, REG_ITMP1);
 632                                 M_INTMOVE(REG_ITMP1, d);
 633                         } else {
 634                                 M_INTMOVE(s1, d);
 635                                 M_ISUB(s2, d);
 636                         }
 637                         emit_store_dst(jd, iptr, d);
 638                         break;
 639
 640                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 641                                       /* sx.val.i = constant                             */
 642
 643                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 644                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 645                         M_INTMOVE(s1, d);
 646                         M_ISUB_IMM(iptr->sx.val.i, d);
 647                         emit_store_dst(jd, iptr, d);
 648                         break;
 649
 650                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 651
 652                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 653                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 654                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 655                         if (s2 == d) {
 656                                 M_INTMOVE(s1, REG_ITMP1);
 657                                 M_LSUB(s2, REG_ITMP1);
 658                                 M_INTMOVE(REG_ITMP1, d);
 659                         } else {
 660                                 M_INTMOVE(s1, d);
 661                                 M_LSUB(s2, d);
 662                         }
 663                         emit_store_dst(jd, iptr, d);
 664                         break;
 665
 666                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 667                                       /* sx.val.l = constant                             */
 668
 669                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 670                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 671                         M_INTMOVE(s1, d);
 672                         if (IS_IMM32(iptr->sx.val.l))
 673                                 M_LSUB_IMM(iptr->sx.val.l, d);
 674                         else {
 675                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 676                                 M_LSUB(REG_ITMP2, d);
 677                         }
 678                         emit_store_dst(jd, iptr, d);
 679                         break;
 680
 681                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 682
 683                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 684                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 685                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 686                         if (s2 == d)
 687                                 M_IMUL(s1, d);
 688                         else {
 689                                 M_INTMOVE(s1, d);
 690                                 M_IMUL(s2, d);
 691                         }
 692                         emit_store_dst(jd, iptr, d);
 693                         break;
 694
 695                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 696                                       /* sx.val.i = constant                             */
 697
 698                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 699                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 700                         if (iptr->sx.val.i == 2) {
 701                                 M_INTMOVE(s1, d);
 702                                 M_ISLL_IMM(1, d);
 703                         } else
 704                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 705                         emit_store_dst(jd, iptr, d);
 706                         break;
 707
 708                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 709
 710                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 711                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 712                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 713                         if (s2 == d)
 714                                 M_LMUL(s1, d);
 715                         else {
 716                                 M_INTMOVE(s1, d);
 717                                 M_LMUL(s2, d);
 718                         }
 719                         emit_store_dst(jd, iptr, d);
 720                         break;
 721
 722                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 723                                       /* sx.val.l = constant                             */
 724
 725                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 726                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 727                         if (IS_IMM32(iptr->sx.val.l))
 728                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 729                         else {
 730                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 731                                 M_INTMOVE(s1, d);
 732                                 M_LMUL(REG_ITMP2, d);
 733                         }
 734                         emit_store_dst(jd, iptr, d);
 735                         break;
 736
 737                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 738
 739                         s1 = emit_load_s1(jd, iptr, RAX);
 740                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 741                         d = codegen_reg_of_dst(jd, iptr, RAX);
 742
 743                         M_INTMOVE(s1, RAX);
 744                         M_INTMOVE(s2, REG_ITMP3);
 745                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 746
 747                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 748
 749                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 750                         M_BNE(4 + 6);
 751                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 752                         M_BEQ(1 + 3);                                          /* 6 bytes */
 753
 754                         emit_cltd(cd);                                         /* 1 byte  */
 755                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 bytes */
 756
 757                         M_INTMOVE(RAX, d);
 758                         emit_store_dst(jd, iptr, d);
 759                         dst = VAROP(iptr->dst);
 760                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 761                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 762                         break;
 763
 764                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 765
 766                         s1 = emit_load_s1(jd, iptr, RAX);
 767                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 768                         d = codegen_reg_of_dst(jd, iptr, RDX);
 769
 770                         M_INTMOVE(s1, RAX);
 771                         M_INTMOVE(s2, REG_ITMP3);
 772                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 773
 774                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 775
 776                         M_ICMP_IMM(0x80000000, RAX);    /* check as described in jvm spec */
 777                         M_BNE(3 + 4 + 6);
 778                         M_CLR(RDX);                                            /* 3 bytes */
 779                         M_ICMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 780                         M_BEQ(1 + 3);                                          /* 6 bytes */
 781
 782                         emit_cltd(cd);                                         /* 1 byte  */
 783                         emit_idivl_reg(cd, REG_ITMP3);                         /* 3 byte  */
 784
 785                         M_INTMOVE(RDX, d);
 786                         emit_store_dst(jd, iptr, d);
 787                         dst = VAROP(iptr->dst);
 788                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 789                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 790                         break;
 791
 792                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 793                                       /* sx.val.i = constant                             */
 794
 795                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 796                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 797                         M_INTMOVE(s1, REG_ITMP1);
 798                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 799                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 800                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 801                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 802                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 803                         emit_store_dst(jd, iptr, d);
 804                         break;
 805
 806                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 807                                       /* sx.val.i = constant                             */
 808
 809                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 810                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 811                         M_INTMOVE(s1, REG_ITMP1);
 812                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 813                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 814                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 815                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 816                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 817                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 818                         emit_store_dst(jd, iptr, d);
 819                         break;
 820
 821
 822                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 823
 824                         s1 = emit_load_s1(jd, iptr, RAX);
 825                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 826                         d = codegen_reg_of_dst(jd, iptr, RAX);
 827
 828                         M_INTMOVE(s1, RAX);
 829                         M_INTMOVE(s2, REG_ITMP3);
 830                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 831
 832                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 833
 834                         /* check as described in jvm spec */
 835                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 836                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 837                         M_BNE(4 + 6);
 838                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 839                         M_BEQ(2 + 3);                                          /* 6 bytes */
 840
 841                         emit_cqto(cd);                                         /* 2 bytes */
 842                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 843
 844                         M_INTMOVE(RAX, d);
 845                         emit_store_dst(jd, iptr, d);
 846                         dst = VAROP(iptr->dst);
 847                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 848                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 849                         break;
 850
 851                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 852
 853                         s1 = emit_load_s1(jd, iptr, RAX);
 854                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 855                         d = codegen_reg_of_dst(jd, iptr, RDX);
 856
 857                         M_INTMOVE(s1, RAX);
 858                         M_INTMOVE(s2, REG_ITMP3);
 859                         emit_arithmetic_check(cd, iptr, REG_ITMP3);
 860
 861                         M_MOV(RDX, REG_ITMP2);    /* save RDX (it's an argument register) */
 862
 863                         /* check as described in jvm spec */
 864                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 865                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 866                         M_BNE(3 + 4 + 6);
 867                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 868                         M_LCMP_IMM(-1, REG_ITMP3);                             /* 4 bytes */
 869                         M_BEQ(2 + 3);                                          /* 6 bytes */
 870
 871                         emit_cqto(cd);                                         /* 2 bytes */
 872                         emit_idiv_reg(cd, REG_ITMP3);                          /* 3 bytes */
 873
 874                         M_INTMOVE(RDX, d);
 875                         emit_store_dst(jd, iptr, d);
 876                         dst = VAROP(iptr->dst);
 877                         if (IS_INMEMORY(dst->flags) || (dst->vv.regoff != RDX))
 878                                 M_MOV(REG_ITMP2, RDX);                         /* restore RDX */
 879                         break;
 880
 881                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 882                                       /* sx.val.i = constant                             */
 883
 884                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 885                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 886                         M_INTMOVE(s1, REG_ITMP1);
 887                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 888                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 889                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 890                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 891                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 892                         emit_store_dst(jd, iptr, d);
 893                         break;
 894
 895                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
 896                                       /* sx.val.l = constant                             */
 897
 898                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 899                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 900                         M_INTMOVE(s1, REG_ITMP1);
 901                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 902                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 903                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 904                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 905                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 906                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 907                         emit_store_dst(jd, iptr, d);
 908                         break;
 909
 910                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 911
 912                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 913                         emit_ishift(jd, SHIFT_SHL, iptr);
 914                         break;
 915
 916                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
 917                                       /* sx.val.i = constant                             */
 918
 919                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 920                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 921                         M_INTMOVE(s1, d);
 922                         M_ISLL_IMM(iptr->sx.val.i, d);
 923                         emit_store_dst(jd, iptr, d);
 924                         break;
 925
 926                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 927
 928                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 929                         emit_ishift(jd, SHIFT_SAR, iptr);
 930                         break;
 931
 932                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
 933                                       /* sx.val.i = constant                             */
 934
 935                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 936                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 937                         M_INTMOVE(s1, d);
 938                         M_ISRA_IMM(iptr->sx.val.i, d);
 939                         emit_store_dst(jd, iptr, d);
 940                         break;
 941
 942                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 943
 944                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 945                         emit_ishift(jd, SHIFT_SHR, iptr);
 946                         break;
 947
 948                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 949                                       /* sx.val.i = constant                             */
 950
 951                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 952                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 953                         M_INTMOVE(s1, d);
 954                         M_ISRL_IMM(iptr->sx.val.i, d);
 955                         emit_store_dst(jd, iptr, d);
 956                         break;
 957
 958                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
 959
 960                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 961                         emit_lshift(jd, SHIFT_SHL, iptr);
 962                         break;
 963
 964         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
 965                                           /* sx.val.i = constant                             */
 966
 967                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 968                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 969                         M_INTMOVE(s1, d);
 970                         M_LSLL_IMM(iptr->sx.val.i, d);
 971                         emit_store_dst(jd, iptr, d);
 972                         break;
 973
 974                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
 975
 976                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 977                         emit_lshift(jd, SHIFT_SAR, iptr);
 978                         break;
 979
 980                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
 981                                       /* sx.val.i = constant                             */
 982
 983                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 984                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 985                         M_INTMOVE(s1, d);
 986                         M_LSRA_IMM(iptr->sx.val.i, d);
 987                         emit_store_dst(jd, iptr, d);
 988                         break;
 989
 990                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
 991
 992                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 993                         emit_lshift(jd, SHIFT_SHR, iptr);
 994                         break;
 995
 996                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
 997                                       /* sx.val.l = constant                             */
 998
 999                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1000                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1001                         M_INTMOVE(s1, d);
1002                         M_LSRL_IMM(iptr->sx.val.i, d);
1003                         emit_store_dst(jd, iptr, d);
1004                         break;
1005
1006                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1007
1008                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1009                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1010                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1011                         if (s2 == d)
1012                                 M_IAND(s1, d);
1013                         else {
1014                                 M_INTMOVE(s1, d);
1015                                 M_IAND(s2, d);
1016                         }
1017                         emit_store_dst(jd, iptr, d);
1018                         break;
1019
1020                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1021                                       /* sx.val.i = constant                             */
1022
1023                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1024                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1025                         M_INTMOVE(s1, d);
1026                         M_IAND_IMM(iptr->sx.val.i, d);
1027                         emit_store_dst(jd, iptr, d);
1028                         break;
1029
1030                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1031
1032                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1033                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1034                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1035                         if (s2 == d)
1036                                 M_LAND(s1, d);
1037                         else {
1038                                 M_INTMOVE(s1, d);
1039                                 M_LAND(s2, d);
1040                         }
1041                         emit_store_dst(jd, iptr, d);
1042                         break;
1043
1044                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1045                                       /* sx.val.l = constant                             */
1046
1047                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1048                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1049                         M_INTMOVE(s1, d);
1050                         if (IS_IMM32(iptr->sx.val.l))
1051                                 M_LAND_IMM(iptr->sx.val.l, d);
1052                         else {
1053                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1054                                 M_LAND(REG_ITMP2, d);
1055                         }
1056                         emit_store_dst(jd, iptr, d);
1057                         break;
1058
1059                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1060
1061                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1062                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1063                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1064                         if (s2 == d)
1065                                 M_IOR(s1, d);
1066                         else {
1067                                 M_INTMOVE(s1, d);
1068                                 M_IOR(s2, d);
1069                         }
1070                         emit_store_dst(jd, iptr, d);
1071                         break;
1072
1073                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1074                                       /* sx.val.i = constant                             */
1075
1076                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1077                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1078                         M_INTMOVE(s1, d);
1079                         M_IOR_IMM(iptr->sx.val.i, d);
1080                         emit_store_dst(jd, iptr, d);
1081                         break;
1082
1083                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1084
1085                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1086                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1087                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1088                         if (s2 == d)
1089                                 M_LOR(s1, d);
1090                         else {
1091                                 M_INTMOVE(s1, d);
1092                                 M_LOR(s2, d);
1093                         }
1094                         emit_store_dst(jd, iptr, d);
1095                         break;
1096
1097                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1098                                       /* sx.val.l = constant                             */
1099
1100                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1101                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1102                         M_INTMOVE(s1, d);
1103                         if (IS_IMM32(iptr->sx.val.l))
1104                                 M_LOR_IMM(iptr->sx.val.l, d);
1105                         else {
1106                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1107                                 M_LOR(REG_ITMP2, d);
1108                         }
1109                         emit_store_dst(jd, iptr, d);
1110                         break;
1111
1112                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1113
1114                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1115                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1116                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1117                         if (s2 == d)
1118                                 M_IXOR(s1, d);
1119                         else {
1120                                 M_INTMOVE(s1, d);
1121                                 M_IXOR(s2, d);
1122                         }
1123                         emit_store_dst(jd, iptr, d);
1124                         break;
1125
1126                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1127                                       /* sx.val.i = constant                             */
1128
1129                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1130                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1131                         M_INTMOVE(s1, d);
1132                         M_IXOR_IMM(iptr->sx.val.i, d);
1133                         emit_store_dst(jd, iptr, d);
1134                         break;
1135
1136                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1137
1138                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1139                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1140                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1141                         if (s2 == d)
1142                                 M_LXOR(s1, d);
1143                         else {
1144                                 M_INTMOVE(s1, d);
1145                                 M_LXOR(s2, d);
1146                         }
1147                         emit_store_dst(jd, iptr, d);
1148                         break;
1149
1150                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1151                                       /* sx.val.l = constant                             */
1152
1153                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1154                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1155                         M_INTMOVE(s1, d);
1156                         if (IS_IMM32(iptr->sx.val.l))
1157                                 M_LXOR_IMM(iptr->sx.val.l, d);
1158                         else {
1159                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1160                                 M_LXOR(REG_ITMP2, d);
1161                         }
1162                         emit_store_dst(jd, iptr, d);
1163                         break;
1164
1165
1166                 /* floating operations ************************************************/
1167
1168                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1169
1170                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1171                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1172                         disp = dseg_add_s4(cd, 0x80000000);
1173                         M_FLTMOVE(s1, d);
1174                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1175                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1176                         emit_store_dst(jd, iptr, d);
1177                         break;
1178
1179                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1180
1181                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1182                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1183                         disp = dseg_add_s8(cd, 0x8000000000000000);
1184                         M_FLTMOVE(s1, d);
1185                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1186                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1187                         emit_store_dst(jd, iptr, d);
1188                         break;
1189
1190                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1191
1192                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1193                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1194                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1195                         if (s2 == d)
1196                                 M_FADD(s1, d);
1197                         else {
1198                                 M_FLTMOVE(s1, d);
1199                                 M_FADD(s2, d);
1200                         }
1201                         emit_store_dst(jd, iptr, d);
1202                         break;
1203
1204                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1205
1206                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1207                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1208                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1209                         if (s2 == d)
1210                                 M_DADD(s1, d);
1211                         else {
1212                                 M_FLTMOVE(s1, d);
1213                                 M_DADD(s2, d);
1214                         }
1215                         emit_store_dst(jd, iptr, d);
1216                         break;
1217
1218                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1219
1220                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1221                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1222                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1223                         if (s2 == d) {
1224                                 M_FLTMOVE(s2, REG_FTMP2);
1225                                 s2 = REG_FTMP2;
1226                         }
1227                         M_FLTMOVE(s1, d);
1228                         M_FSUB(s2, d);
1229                         emit_store_dst(jd, iptr, d);
1230                         break;
1231
1232                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1233
1234                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1235                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1236                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1237                         if (s2 == d) {
1238                                 M_FLTMOVE(s2, REG_FTMP2);
1239                                 s2 = REG_FTMP2;
1240                         }
1241                         M_FLTMOVE(s1, d);
1242                         M_DSUB(s2, d);
1243                         emit_store_dst(jd, iptr, d);
1244                         break;
1245
1246                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1247
1248                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1249                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1250                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1251                         if (s2 == d)
1252                                 M_FMUL(s1, d);
1253                         else {
1254                                 M_FLTMOVE(s1, d);
1255                                 M_FMUL(s2, d);
1256                         }
1257                         emit_store_dst(jd, iptr, d);
1258                         break;
1259
1260                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1261
1262                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1263                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1264                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1265                         if (s2 == d)
1266                                 M_DMUL(s1, d);
1267                         else {
1268                                 M_FLTMOVE(s1, d);
1269                                 M_DMUL(s2, d);
1270                         }
1271                         emit_store_dst(jd, iptr, d);
1272                         break;
1273
1274                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1275
1276                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1277                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1278                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1279                         if (s2 == d) {
1280                                 M_FLTMOVE(s2, REG_FTMP2);
1281                                 s2 = REG_FTMP2;
1282                         }
1283                         M_FLTMOVE(s1, d);
1284                         M_FDIV(s2, d);
1285                         emit_store_dst(jd, iptr, d);
1286                         break;
1287
1288                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1289
1290                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1291                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1292                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1293                         if (s2 == d) {
1294                                 M_FLTMOVE(s2, REG_FTMP2);
1295                                 s2 = REG_FTMP2;
1296                         }
1297                         M_FLTMOVE(s1, d);
1298                         M_DDIV(s2, d);
1299                         emit_store_dst(jd, iptr, d);
1300                         break;
1301
1302                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1303
1304                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1305                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1306                         M_CVTIF(s1, d);
1307                         emit_store_dst(jd, iptr, d);
1308                         break;
1309
1310                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1311
1312                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1313                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1314                         M_CVTID(s1, d);
1315                         emit_store_dst(jd, iptr, d);
1316                         break;
1317
1318                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1319
1320                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1321                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1322                         M_CVTLF(s1, d);
1323                         emit_store_dst(jd, iptr, d);
1324                         break;
1325
1326                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1327
1328                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1329                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1330                         M_CVTLD(s1, d);
1331                         emit_store_dst(jd, iptr, d);
1332                         break;
1333
1334                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1335
1336                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1337                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1338                         M_CVTFI(s1, d);
1339                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1340                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1341                                 ((REG_RESULT == d) ? 0 : 3);
1342                         M_BNE(disp);
1343                         M_FLTMOVE(s1, REG_FTMP1);
1344                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1345                         M_CALL(REG_ITMP2);
1346                         M_INTMOVE(REG_RESULT, d);
1347                         emit_store_dst(jd, iptr, d);
1348                         break;
1349
1350                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1351
1352                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1353                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1354                         M_CVTDI(s1, d);
1355                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1356                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1357                                 ((REG_RESULT == d) ? 0 : 3);
1358                         M_BNE(disp);
1359                         M_FLTMOVE(s1, REG_FTMP1);
1360                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1361                         M_CALL(REG_ITMP2);
1362                         M_INTMOVE(REG_RESULT, d);
1363                         emit_store_dst(jd, iptr, d);
1364                         break;
1365
1366                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1367
1368                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1369                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1370                         M_CVTFL(s1, d);
1371                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1372                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1373                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1374                                 ((REG_RESULT == d) ? 0 : 3);
1375                         M_BNE(disp);
1376                         M_FLTMOVE(s1, REG_FTMP1);
1377                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1378                         M_CALL(REG_ITMP2);
1379                         M_INTMOVE(REG_RESULT, d);
1380                         emit_store_dst(jd, iptr, d);
1381                         break;
1382
1383                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1384
1385                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1386                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1387                         M_CVTDL(s1, d);
1388                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1389                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1390                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1391                                 ((REG_RESULT == d) ? 0 : 3);
1392                         M_BNE(disp);
1393                         M_FLTMOVE(s1, REG_FTMP1);
1394                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1395                         M_CALL(REG_ITMP2);
1396                         M_INTMOVE(REG_RESULT, d);
1397                         emit_store_dst(jd, iptr, d);
1398                         break;
1399
1400                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1401
1402                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1403                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1404                         M_CVTFD(s1, d);
1405                         emit_store_dst(jd, iptr, d);
1406                         break;
1407
1408                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1409
1410                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1411                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1412                         M_CVTDF(s1, d);
1413                         emit_store_dst(jd, iptr, d);
1414                         break;
1415
1416                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1417                                           /* == => 0, < => 1, > => -1 */
1418
1419                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1420                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1421                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1422                         M_CLR(d);
1423                         M_MOV_IMM(1, REG_ITMP1);
1424                         M_MOV_IMM(-1, REG_ITMP2);
1425                         emit_ucomiss_reg_reg(cd, s1, s2);
1426                         M_CMOVULT(REG_ITMP1, d);
1427                         M_CMOVUGT(REG_ITMP2, d);
1428                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1429                         emit_store_dst(jd, iptr, d);
1430                         break;
1431
1432                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1433                                           /* == => 0, < => 1, > => -1 */
1434
1435                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1436                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1437                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1438                         M_CLR(d);
1439                         M_MOV_IMM(1, REG_ITMP1);
1440                         M_MOV_IMM(-1, REG_ITMP2);
1441                         emit_ucomiss_reg_reg(cd, s1, s2);
1442                         M_CMOVULT(REG_ITMP1, d);
1443                         M_CMOVUGT(REG_ITMP2, d);
1444                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1445                         emit_store_dst(jd, iptr, d);
1446                         break;
1447
1448                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1449                                           /* == => 0, < => 1, > => -1 */
1450
1451                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1452                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1453                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1454                         M_CLR(d);
1455                         M_MOV_IMM(1, REG_ITMP1);
1456                         M_MOV_IMM(-1, REG_ITMP2);
1457                         emit_ucomisd_reg_reg(cd, s1, s2);
1458                         M_CMOVULT(REG_ITMP1, d);
1459                         M_CMOVUGT(REG_ITMP2, d);
1460                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1461                         emit_store_dst(jd, iptr, d);
1462                         break;
1463
1464                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1465                                           /* == => 0, < => 1, > => -1 */
1466
1467                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1468                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1469                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1470                         M_CLR(d);
1471                         M_MOV_IMM(1, REG_ITMP1);
1472                         M_MOV_IMM(-1, REG_ITMP2);
1473                         emit_ucomisd_reg_reg(cd, s1, s2);
1474                         M_CMOVULT(REG_ITMP1, d);
1475                         M_CMOVUGT(REG_ITMP2, d);
1476                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1477                         emit_store_dst(jd, iptr, d);
1478                         break;
1479
1480
1481                 /* memory operations **************************************************/
1482
1483                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1484
1485                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1486                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1487                         /* implicit null-pointer check */
1488                         M_ILD(d, s1, OFFSET(java_array_t, size));
1489                         emit_store_dst(jd, iptr, d);
1490                         break;
1491
1492                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1493
1494                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1495                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1496                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1497                         /* implicit null-pointer check */
1498                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1499                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray_t, data[0]), s1, s2, 0, d);
1500                         emit_store_dst(jd, iptr, d);
1501                         break;
1502
1503                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1504
1505                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1506                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1507                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1508                         /* implicit null-pointer check */
1509                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1510                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray_t, data[0]), s1, s2, 1, d);
1511                         emit_store_dst(jd, iptr, d);
1512                         break;
1513
1514                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1515
1516                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1517                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1518                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1519                         /* implicit null-pointer check */
1520                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1521                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray_t, data[0]), s1, s2, 1, d);
1522                         emit_store_dst(jd, iptr, d);
1523                         break;
1524
1525                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1526
1527                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1528                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1529                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1530                         /* implicit null-pointer check */
1531                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1532                         emit_movl_memindex_reg(cd, OFFSET(java_intarray_t, data[0]), s1, s2, 2, d);
1533                         emit_store_dst(jd, iptr, d);
1534                         break;
1535
1536                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1537
1538                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1539                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1540                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1541                         /* implicit null-pointer check */
1542                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1543                         emit_mov_memindex_reg(cd, OFFSET(java_longarray_t, data[0]), s1, s2, 3, d);
1544                         emit_store_dst(jd, iptr, d);
1545                         break;
1546
1547                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1548
1549                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1550                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1551                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1552                         /* implicit null-pointer check */
1553                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1554                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray_t, data[0]), s1, s2, 2, d);
1555                         emit_store_dst(jd, iptr, d);
1556                         break;
1557
1558                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1559
1560                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1561                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1562                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1563                         /* implicit null-pointer check */
1564                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1565                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray_t, data[0]), s1, s2, 3, d);
1566                         emit_store_dst(jd, iptr, d);
1567                         break;
1568
1569                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1570
1571                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1572                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1573                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1574                         /* implicit null-pointer check */
1575                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1576                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray_t, data[0]), s1, s2, 3, d);
1577                         emit_store_dst(jd, iptr, d);
1578                         break;
1579
1580
1581                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1582
1583                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1584                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1585                         /* implicit null-pointer check */
1586                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1587                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1588                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1589                         break;
1590
1591                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1592
1593                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1594                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1595                         /* implicit null-pointer check */
1596                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1597                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1598                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1599                         break;
1600
1601                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1602
1603                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1604                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1605                         /* implicit null-pointer check */
1606                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1607                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1608                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1609                         break;
1610
1611                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1612
1613                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1614                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1615                         /* implicit null-pointer check */
1616                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1617                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1618                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1619                         break;
1620
1621                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1622
1623                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1624                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1625                         /* implicit null-pointer check */
1626                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1627                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1628                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1629                         break;
1630
1631                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1632
1633                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1634                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1635                         /* implicit null-pointer check */
1636                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1637                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1638                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray_t, data[0]), s1, s2, 2);
1639                         break;
1640
1641                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1642
1643                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1644                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1645                         /* implicit null-pointer check */
1646                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1647                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1648                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray_t, data[0]), s1, s2, 3);
1649                         break;
1650
1651                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1652
1653                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1654                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1655                         /* implicit null-pointer check */
1656                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1657                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1658
1659                         M_MOV(s1, REG_A0);
1660                         M_MOV(s3, REG_A1);
1661                         M_MOV_IMM(BUILTIN_FAST_canstore, REG_ITMP1);
1662                         M_CALL(REG_ITMP1);
1663                         emit_arraystore_check(cd, iptr);
1664
1665                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1666                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1667                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1668                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1669                         break;
1670
1671
1672                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1673
1674                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1675                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1676                         /* implicit null-pointer check */
1677                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1678                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray_t, data[0]), s1, s2, 0);
1679                         break;
1680
1681                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1682
1683                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1684                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1685                         /* implicit null-pointer check */
1686                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1687                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray_t, data[0]), s1, s2, 1);
1688                         break;
1689
1690                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1691
1692                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1693                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1694                         /* implicit null-pointer check */
1695                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1696                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray_t, data[0]), s1, s2, 1);
1697                         break;
1698
1699                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1700
1701                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1702                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1703                         /* implicit null-pointer check */
1704                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1705                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray_t, data[0]), s1, s2, 2);
1706                         break;
1707
1708                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1709
1710                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1711                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1712                         /* implicit null-pointer check */
1713                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1714
1715                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1716                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1717                         }
1718                         else {
1719                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray_t, data[0]), s1, s2, 3);
1720                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray_t, data[0]) + 4, s1, s2, 3);
1721                         }
1722                         break;
1723
1724                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1725
1726                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1727                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1728                         /* implicit null-pointer check */
1729                         emit_arrayindexoutofbounds_check(cd, iptr, s1, s2);
1730                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray_t, data[0]), s1, s2, 3);
1731                         break;
1732
1733
1734                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1735
1736                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1737                                 uf        = iptr->sx.s23.s3.uf;
1738                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1739                                 disp      = dseg_add_unique_address(cd, uf);
1740
1741 /*                              PROFILE_CYCLE_STOP; */
1742
1743                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1744
1745 /*                              PROFILE_CYCLE_START; */
1746                         }
1747                         else {
1748                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1749                                 fieldtype = fi->type;
1750                                 disp      = dseg_add_address(cd, fi->value);
1751
1752                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1753                                         PROFILE_CYCLE_STOP;
1754
1755                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1756                                                                                   fi->clazz, 0);
1757
1758                                         PROFILE_CYCLE_START;
1759                                 }
1760                         }
1761
1762                         /* This approach is much faster than moving the field
1763                            address inline into a register. */
1764
1765                         M_ALD(REG_ITMP1, RIP, disp);
1766
1767                         switch (fieldtype) {
1768                         case TYPE_INT:
1769                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1770                                 M_ILD(d, REG_ITMP1, 0);
1771                                 break;
1772                         case TYPE_LNG:
1773                         case TYPE_ADR:
1774                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1775                                 M_LLD(d, REG_ITMP1, 0);
1776                                 break;
1777                         case TYPE_FLT:
1778                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1779                                 M_FLD(d, REG_ITMP1, 0);
1780                                 break;
1781                         case TYPE_DBL:
1782                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1783                                 M_DLD(d, REG_ITMP1, 0);
1784                                 break;
1785                         }
1786                         emit_store_dst(jd, iptr, d);
1787                         break;
1788
1789                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1790
1791                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1792                                 uf        = iptr->sx.s23.s3.uf;
1793                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1794                                 disp      = dseg_add_unique_address(cd, uf);
1795
1796 /*                              PROFILE_CYCLE_STOP; */
1797
1798                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1799
1800 /*                              PROFILE_CYCLE_START; */
1801                         }
1802                         else {
1803                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1804                                 fieldtype = fi->type;
1805                                 disp      = dseg_add_address(cd, fi->value);
1806
1807                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1808                                         PROFILE_CYCLE_STOP;
1809
1810                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1811                                                                                   fi->clazz, 0);
1812
1813                                         PROFILE_CYCLE_START;
1814                                 }
1815                         }
1816
1817                         /* This approach is much faster than moving the field
1818                            address inline into a register. */
1819
1820                         M_ALD(REG_ITMP1, RIP, disp);
1821
1822                         switch (fieldtype) {
1823                         case TYPE_INT:
1824                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1825                                 M_IST(s1, REG_ITMP1, 0);
1826                                 break;
1827                         case TYPE_LNG:
1828                         case TYPE_ADR:
1829                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1830                                 M_LST(s1, REG_ITMP1, 0);
1831                                 break;
1832                         case TYPE_FLT:
1833                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1834                                 M_FST(s1, REG_ITMP1, 0);
1835                                 break;
1836                         case TYPE_DBL:
1837                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1838                                 M_DST(s1, REG_ITMP1, 0);
1839                                 break;
1840                         }
1841                         break;
1842
1843                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
1844                                           /* val = value (in current instruction)     */
1845                                           /* following NOP)                           */
1846
1847                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1848                                 uf        = iptr->sx.s23.s3.uf;
1849                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1850                                 disp      = dseg_add_unique_address(cd, uf);
1851
1852 /*                              PROFILE_CYCLE_STOP; */
1853
1854                                 patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
1855
1856 /*                              PROFILE_CYCLE_START; */
1857                         }
1858                         else {
1859                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1860                                 fieldtype = fi->type;
1861                                 disp      = dseg_add_address(cd, fi->value);
1862
1863                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->clazz)) {
1864                                         PROFILE_CYCLE_STOP;
1865
1866                                         patcher_add_patch_ref(jd, PATCHER_initialize_class,
1867                                                                                   fi->clazz, 0);
1868
1869                                         PROFILE_CYCLE_START;
1870                                 }
1871                         }
1872
1873                         /* This approach is much faster than moving the field
1874                            address inline into a register. */
1875
1876                         M_ALD(REG_ITMP1, RIP, disp);
1877
1878                         switch (fieldtype) {
1879                         case TYPE_INT:
1880                         case TYPE_FLT:
1881                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1882                                 break;
1883                         case TYPE_LNG:
1884                         case TYPE_ADR:
1885                         case TYPE_DBL:
1886                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
1887                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
1888                                 else {
1889                                         M_MOV_IMM(iptr->sx.s23.s2.constval, REG_ITMP2);
1890                                         M_LST(REG_ITMP2, REG_ITMP1, 0);
1891                                 }
1892                                 break;
1893                         }
1894                         break;
1895
1896                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
1897
1898                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1899
1900                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1901                                 uf        = iptr->sx.s23.s3.uf;
1902                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1903                                 disp      = 0;
1904
1905 /*                              PROFILE_CYCLE_STOP; */
1906
1907                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1908
1909 /*                              PROFILE_CYCLE_START; */
1910                         }
1911                         else {
1912                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1913                                 fieldtype = fi->type;
1914                                 disp      = fi->offset;
1915                         }
1916
1917                         /* implicit null-pointer check */
1918                         switch (fieldtype) {
1919                         case TYPE_INT:
1920                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1921                                 M_ILD32(d, s1, disp);
1922                                 break;
1923                         case TYPE_LNG:
1924                         case TYPE_ADR:
1925                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1926                                 M_LLD32(d, s1, disp);
1927                                 break;
1928                         case TYPE_FLT:
1929                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1930                                 M_FLD32(d, s1, disp);
1931                                 break;
1932                         case TYPE_DBL:
1933                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1934                                 M_DLD32(d, s1, disp);
1935                                 break;
1936                         }
1937                         emit_store_dst(jd, iptr, d);
1938                         break;
1939
1940                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
1941
1942                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1943                         s2 = emit_load_s2(jd, iptr, REG_IFTMP); /* REG_IFTMP == REG_ITMP2 */
1944
1945                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1946                                 uf        = iptr->sx.s23.s3.uf;
1947                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1948                                 disp      = 0;
1949
1950 /*                              PROFILE_CYCLE_STOP; */
1951
1952                                 patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
1953
1954 /*                              PROFILE_CYCLE_START; */
1955                         }
1956                         else {
1957                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1958                                 fieldtype = fi->type;
1959                                 disp      = fi->offset;
1960                         }
1961
1962                         /* implicit null-pointer check */
1963                         switch (fieldtype) {
1964                         case TYPE_INT:
1965                                 M_IST32(s2, s1, disp);
1966                                 break;
1967                         case TYPE_LNG:
1968                         case TYPE_ADR:
1969                                 M_LST32(s2, s1, disp);
1970                                 break;
1971                         case TYPE_FLT:
1972                                 M_FST32(s2, s1, disp);
1973                                 break;
1974                         case TYPE_DBL:
1975                                 M_DST32(s2, s1, disp);
1976                                 break;
1977                         }
1978                         break;
1979
1980                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
1981                                           /* val = value (in current instruction)     */
1982                                           /* following NOP)                           */
1983
1984                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1985
1986                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1987                                 uf        = iptr->sx.s23.s3.uf;
1988                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1989                                 disp      = 0;
1990
1991 /*                              PROFILE_CYCLE_STOP; */
1992
1993                                 patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
1994
1995 /*                              PROFILE_CYCLE_START; */
1996                         }
1997                         else {
1998                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1999                                 fieldtype = fi->type;
2000                                 disp      = fi->offset;
2001                         }
2002
2003                         /* implicit null-pointer check */
2004                         switch (fieldtype) {
2005                         case TYPE_INT:
2006                         case TYPE_FLT:
2007                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2008                                 break;
2009                         case TYPE_LNG:
2010                         case TYPE_ADR:
2011                         case TYPE_DBL:
2012                                 /* XXX why no check for IS_IMM32? -- probably because of the patcher */
2013                                 M_MOV_IMM(iptr->sx.s23.s2.constval, REG_ITMP2);
2014                                 if (disp)  /* resolved, disp can never be 0 */
2015                                         M_LST(REG_ITMP2, s1, disp);
2016                                 else       /* unresolved */
2017                                         M_LST32(REG_ITMP2, s1, disp);
2018                                 break;
2019                         }
2020                         break;
2021
2022
2023                 /* branch operations **************************************************/
2024
2025                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2026
2027                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2028                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2029
2030                         PROFILE_CYCLE_STOP;
2031
2032 #ifdef ENABLE_VERIFIER
2033                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2034                                 uc = iptr->sx.s23.s2.uc;
2035
2036                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2037                         }
2038 #endif /* ENABLE_VERIFIER */
2039
2040                         M_CALL_IMM(0);                            /* passing exception pc */
2041                         M_POP(REG_ITMP2_XPC);
2042
2043                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2044                         M_JMP(REG_ITMP3);
2045                         break;
2046
2047                 case ICMD_GOTO:         /* ... ==> ...                                */
2048                 case ICMD_RET:
2049
2050                         emit_br(cd, iptr->dst.block);
2051                         ALIGNCODENOP;
2052                         break;
2053
2054                 case ICMD_JSR:          /* ... ==> ...                                */
2055
2056                         emit_br(cd, iptr->sx.s23.s3.jsrtarget.block);
2057                         ALIGNCODENOP;
2058                         break;
2059
2060                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2061                 case ICMD_IFNONNULL:
2062
2063                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2064                         M_TEST(s1);
2065                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFNULL, BRANCH_OPT_NONE);
2066                         break;
2067
2068                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2069                 case ICMD_IFLT:
2070                 case ICMD_IFLE:
2071                 case ICMD_IFNE:
2072                 case ICMD_IFGT:
2073                 case ICMD_IFGE:
2074
2075                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2076                         M_ICMP_IMM(iptr->sx.val.i, s1);
2077                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IFEQ, BRANCH_OPT_NONE);
2078                         break;
2079
2080                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2081                 case ICMD_IF_LNE:
2082                 case ICMD_IF_LLT:
2083                 case ICMD_IF_LGE:
2084                 case ICMD_IF_LGT:
2085                 case ICMD_IF_LLE:
2086
2087                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2088                         if (IS_IMM32(iptr->sx.val.l))
2089                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2090                         else {
2091                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2092                                 M_LCMP(REG_ITMP2, s1);
2093                         }
2094                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LEQ, BRANCH_OPT_NONE);
2095                         break;
2096
2097                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2098                 case ICMD_IF_ICMPNE:
2099                 case ICMD_IF_ICMPLT:
2100                 case ICMD_IF_ICMPGE:
2101                 case ICMD_IF_ICMPGT:
2102                 case ICMD_IF_ICMPLE:
2103
2104                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2105                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2106                         M_ICMP(s2, s1);
2107                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ICMPEQ, BRANCH_OPT_NONE);
2108                         break;
2109
2110                 case ICMD_IF_ACMPEQ:    /* ..., value, value ==> ...                  */
2111                 case ICMD_IF_ACMPNE:
2112
2113                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2114                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2115                         M_LCMP(s2, s1);
2116                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_ACMPEQ, BRANCH_OPT_NONE);
2117                         break;
2118
2119                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2120                 case ICMD_IF_LCMPNE:
2121                 case ICMD_IF_LCMPLT:
2122                 case ICMD_IF_LCMPGE:
2123                 case ICMD_IF_LCMPGT:
2124                 case ICMD_IF_LCMPLE:
2125
2126                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2127                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2128                         M_LCMP(s2, s1);
2129                         emit_bcc(cd, iptr->dst.block, iptr->opc - ICMD_IF_LCMPEQ, BRANCH_OPT_NONE);
2130                         break;
2131
2132                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2133                 case ICMD_LRETURN:
2134
2135                         REPLACEMENT_POINT_RETURN(cd, iptr);
2136                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2137                         M_INTMOVE(s1, REG_RESULT);
2138                         goto nowperformreturn;
2139
2140                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2141
2142                         REPLACEMENT_POINT_RETURN(cd, iptr);
2143                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2144                         M_INTMOVE(s1, REG_RESULT);
2145
2146 #ifdef ENABLE_VERIFIER
2147                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2148                                 uc = iptr->sx.s23.s2.uc;
2149
2150                                 PROFILE_CYCLE_STOP;
2151
2152                                 patcher_add_patch_ref(jd, PATCHER_resolve_class, uc, 0);
2153
2154                                 PROFILE_CYCLE_START;
2155                         }
2156 #endif /* ENABLE_VERIFIER */
2157                         goto nowperformreturn;
2158
2159                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2160                 case ICMD_DRETURN:
2161
2162                         REPLACEMENT_POINT_RETURN(cd, iptr);
2163                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2164                         M_FLTMOVE(s1, REG_FRESULT);
2165                         goto nowperformreturn;
2166
2167                 case ICMD_RETURN:      /* ...  ==> ...                                */
2168
2169                         REPLACEMENT_POINT_RETURN(cd, iptr);
2170
2171 nowperformreturn:
2172                         {
2173                         s4 i, p;
2174
2175                         p = cd->stackframesize;
2176
2177 #if !defined(NDEBUG)
2178                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2179                                 emit_verbosecall_exit(jd);
2180 #endif /* !defined(NDEBUG) */
2181
2182 #if defined(ENABLE_THREADS)
2183                         if (checksync && code_is_synchronized(code)) {
2184                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2185
2186                                 /* we need to save the proper return value */
2187                                 switch (iptr->opc) {
2188                                 case ICMD_IRETURN:
2189                                 case ICMD_ARETURN:
2190                                 case ICMD_LRETURN:
2191                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2192                                         break;
2193                                 case ICMD_FRETURN:
2194                                 case ICMD_DRETURN:
2195                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2196                                         break;
2197                                 }
2198
2199                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2200                                 M_CALL(REG_ITMP1);
2201
2202                                 /* and now restore the proper return value */
2203                                 switch (iptr->opc) {
2204                                 case ICMD_IRETURN:
2205                                 case ICMD_ARETURN:
2206                                 case ICMD_LRETURN:
2207                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2208                                         break;
2209                                 case ICMD_FRETURN:
2210                                 case ICMD_DRETURN:
2211                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2212                                         break;
2213                                 }
2214                         }
2215 #endif
2216
2217                         /* restore saved registers */
2218
2219                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2220                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2221                         }
2222                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2223                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2224                         }
2225
2226                         /* deallocate stack */
2227
2228                         if (cd->stackframesize)
2229                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2230
2231                         /* generate method profiling code */
2232
2233                         PROFILE_CYCLE_STOP;
2234
2235                         M_RET;
2236                         }
2237                         break;
2238
2239
2240                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2241                         {
2242                                 s4 i, l;
2243                                 branch_target_t *table;
2244
2245                                 table = iptr->dst.table;
2246
2247                                 l = iptr->sx.s23.s2.tablelow;
2248                                 i = iptr->sx.s23.s3.tablehigh;
2249
2250                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2251                                 M_INTMOVE(s1, REG_ITMP1);
2252
2253                                 if (l != 0)
2254                                         M_ISUB_IMM(l, REG_ITMP1);
2255
2256                                 /* number of targets */
2257                                 i = i - l + 1;
2258
2259                 /* range check */
2260
2261                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2262                                 emit_bugt(cd, table[0].block);
2263
2264                                 /* build jump table top down and use address of lowest entry */
2265
2266                                 table += i;
2267
2268                                 while (--i >= 0) {
2269                                         dseg_add_target(cd, table->block);
2270                                         --table;
2271                                 }
2272
2273                                 /* length of dataseg after last dseg_add_target is used
2274                                    by load */
2275
2276                                 M_MOV_IMM(0, REG_ITMP2);
2277                                 dseg_adddata(cd);
2278                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2279                                 M_JMP(REG_ITMP1);
2280                         }
2281                         break;
2282
2283
2284                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2285                         {
2286                                 s4 i;
2287                                 lookup_target_t *lookup;
2288
2289                                 lookup = iptr->dst.lookup;
2290
2291                                 i = iptr->sx.s23.s2.lookupcount;
2292
2293                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2294                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2295
2296                                 while (--i >= 0) {
2297                                         M_ICMP_IMM(lookup->value, s1);
2298                                         emit_beq(cd, lookup->target.block);
2299                                         lookup++;
2300                                 }
2301
2302                                 emit_br(cd, iptr->sx.s23.s3.lookupdefault.block);
2303                                 ALIGNCODENOP;
2304                         }
2305                         break;
2306
2307
2308                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2309
2310                         REPLACEMENT_POINT_FORGC_BUILTIN(cd, iptr);
2311
2312                         bte = iptr->sx.s23.s3.bte;
2313                         md  = bte->md;
2314                         goto gen_method;
2315
2316                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2317
2318                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2319                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2320                 case ICMD_INVOKEINTERFACE:
2321
2322                         REPLACEMENT_POINT_INVOKE(cd, iptr);
2323
2324                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2325                                 lm = NULL;
2326                                 um = iptr->sx.s23.s3.um;
2327                                 md = um->methodref->parseddesc.md;
2328                         }
2329                         else {
2330                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2331                                 um = NULL;
2332                                 md = lm->parseddesc;
2333                         }
2334
2335 gen_method:
2336                         s3 = md->paramcount;
2337
2338                         MCODECHECK((20 * s3) + 128);
2339
2340                         /* copy arguments to registers or stack location */
2341
2342                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2343                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2344                                 d   = md->params[s3].regoff;
2345
2346                                 /* already preallocated (ARGVAR)? */
2347
2348                                 if (var->flags & PREALLOC)
2349                                         continue;
2350
2351                                 if (IS_INT_LNG_TYPE(var->type)) {
2352                                         if (!md->params[s3].inmemory) {
2353                                                 s1 = emit_load(jd, iptr, var, d);
2354                                                 M_INTMOVE(s1, d);
2355                                         }
2356                                         else {
2357                                                 s1 = emit_load(jd, iptr, var, REG_ITMP1);
2358                                                 M_LST(s1, REG_SP, d);
2359                                         }
2360                                 }
2361                                 else {
2362                                         if (!md->params[s3].inmemory) {
2363                                                 s1 = emit_load(jd, iptr, var, d);
2364                                                 M_FLTMOVE(s1, d);
2365                                         }
2366                                         else {
2367                                                 s1 = emit_load(jd, iptr, var, REG_FTMP1);
2368
2369                                                 if (IS_2_WORD_TYPE(var->type))
2370                                                         M_DST(s1, REG_SP, d);
2371                                                 else
2372                                                         M_FST(s1, REG_SP, d);
2373                                         }
2374                                 }
2375                         }
2376
2377                         /* generate method profiling code */
2378
2379                         PROFILE_CYCLE_STOP;
2380
2381                         switch (iptr->opc) {
2382                         case ICMD_BUILTIN:
2383                                 if (bte->stub == NULL) {
2384                                         M_MOV_IMM(bte->fp, REG_ITMP1);
2385                                 }
2386                                 else {
2387                                         M_MOV_IMM(bte->stub, REG_ITMP1);
2388                                 }
2389                                 M_CALL(REG_ITMP1);
2390                                 break;
2391
2392                         case ICMD_INVOKESPECIAL:
2393                                 emit_nullpointer_check(cd, iptr, REG_A0);
2394                                 /* fall through */
2395
2396                         case ICMD_INVOKESTATIC:
2397                                 if (lm == NULL) {
2398                                         disp = dseg_add_unique_address(cd, um);
2399
2400                                         patcher_add_patch_ref(jd, PATCHER_invokestatic_special,
2401                                                                                   um, disp);
2402                                 }
2403                                 else {
2404                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2405                                 }
2406
2407                                 M_ALD(REG_ITMP2, RIP, disp);
2408                                 M_CALL(REG_ITMP2);
2409                                 break;
2410
2411                         case ICMD_INVOKEVIRTUAL:
2412                                 if (lm == NULL) {
2413                                         patcher_add_patch_ref(jd, PATCHER_invokevirtual, um, 0);
2414
2415                                         s1 = 0;
2416                                 }
2417                                 else {
2418                                         s1 = OFFSET(vftbl_t, table[0]) +
2419                                                 sizeof(methodptr) * lm->vftblindex;
2420                                 }
2421
2422                                 /* implicit null-pointer check */
2423                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2424                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2425                                 M_CALL(REG_ITMP3);
2426                                 break;
2427
2428                         case ICMD_INVOKEINTERFACE:
2429                                 if (lm == NULL) {
2430                                         patcher_add_patch_ref(jd, PATCHER_invokeinterface, um, 0);
2431
2432                                         s1 = 0;
2433                                         s2 = 0;
2434                                 }
2435                                 else {
2436                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2437                                                 sizeof(methodptr) * lm->clazz->index;
2438
2439                                         s2 = sizeof(methodptr) * (lm - lm->clazz->methods);
2440                                 }
2441
2442                                 /* implicit null-pointer check */
2443                                 M_ALD(REG_METHODPTR, REG_A0, OFFSET(java_object_t, vftbl));
2444                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2445                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2446                                 M_CALL(REG_ITMP3);
2447                                 break;
2448                         }
2449
2450                         /* generate method profiling code */
2451
2452                         PROFILE_CYCLE_START;
2453
2454                         /* store size of call code in replacement point */
2455
2456                         REPLACEMENT_POINT_INVOKE_RETURN(cd, iptr);
2457                         REPLACEMENT_POINT_FORGC_BUILTIN_RETURN(cd, iptr);
2458
2459                         /* store return value */
2460
2461                         switch (md->returntype.type) {
2462                         case TYPE_INT:
2463                         case TYPE_LNG:
2464                         case TYPE_ADR:
2465                                 s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2466                                 M_INTMOVE(REG_RESULT, s1);
2467                                 emit_store_dst(jd, iptr, s1);
2468                                 break;
2469                         case TYPE_FLT:
2470                         case TYPE_DBL:
2471                                 s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2472                                 M_FLTMOVE(REG_FRESULT, s1);
2473                                 emit_store_dst(jd, iptr, s1);
2474                                 break;
2475                         default:
2476                                 /* TYPE_VOID */
2477                                 break;
2478                         }
2479                         break;
2480
2481
2482                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2483
2484                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2485                                 /* object type cast-check */
2486
2487                                 classinfo *super;
2488                                 s4         superindex;
2489
2490                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2491                                         super      = NULL;
2492                                         superindex = 0;
2493                                 }
2494                                 else {
2495                                         super      = iptr->sx.s23.s3.c.cls;
2496                                         superindex = super->index;
2497                                 }
2498
2499                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2500
2501                                 /* if class is not resolved, check which code to call */
2502
2503                                 if (super == NULL) {
2504                                         M_TEST(s1);
2505                                         emit_label_beq(cd, BRANCH_LABEL_1);
2506
2507                                         patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2508                                                                                   iptr->sx.s23.s3.c.ref, 0);
2509
2510                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2511                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2512                                         emit_label_beq(cd, BRANCH_LABEL_2);
2513                                 }
2514
2515                                 /* interface checkcast code */
2516
2517                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2518                                         if (super != NULL) {
2519                                                 M_TEST(s1);
2520                                                 emit_label_beq(cd, BRANCH_LABEL_3);
2521                                         }
2522
2523                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2524
2525                                         if (super == NULL) {
2526                                                 patcher_add_patch_ref(jd, PATCHER_checkcast_interface,
2527                                                                                           iptr->sx.s23.s3.c.ref,
2528                                                                                           0);
2529                                         }
2530
2531                                         M_ILD32(REG_ITMP3,
2532                                                         REG_ITMP2, OFFSET(vftbl_t, interfacetablelength));
2533                                         M_ICMP_IMM32(superindex, REG_ITMP3);
2534                                         emit_classcast_check(cd, iptr, BRANCH_LE, REG_ITMP3, s1);
2535
2536                                         M_ALD32(REG_ITMP3, REG_ITMP2,
2537                                                         OFFSET(vftbl_t, interfacetable[0]) -
2538                                                         superindex * sizeof(methodptr*));
2539                                         M_TEST(REG_ITMP3);
2540                                         emit_classcast_check(cd, iptr, BRANCH_EQ, REG_ITMP3, s1);
2541
2542                                         if (super == NULL)
2543                                                 emit_label_br(cd, BRANCH_LABEL_4);
2544                                         else
2545                                                 emit_label(cd, BRANCH_LABEL_3);
2546                                 }
2547
2548                                 /* class checkcast code */
2549
2550                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2551                                         if (super == NULL) {
2552                                                 emit_label(cd, BRANCH_LABEL_2);
2553
2554                                                 cr   = iptr->sx.s23.s3.c.ref;
2555                                                 disp = dseg_add_unique_address(cd, cr);
2556
2557                                                 patcher_add_patch_ref(jd,
2558                                                                                           PATCHER_resolve_classref_to_vftbl,
2559                                                                                           cr, disp);
2560                                         }
2561                                         else {
2562                                                 M_TEST(s1);
2563                                                 emit_label_beq(cd, BRANCH_LABEL_5);
2564
2565                                                 disp = dseg_add_address(cd, super->vftbl);
2566                                         }
2567
2568                                         M_ALD(REG_ITMP2, s1, OFFSET(java_object_t, vftbl));
2569                                         M_ALD(REG_ITMP3, RIP, disp);
2570
2571                                         M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2572
2573                                         /*                                      if (s1 != REG_ITMP1) { */
2574                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2575                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2576                                         /*                                                                                              REG_ITMP1); */
2577                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2578                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
2579                                         /*                                                                                              REG_ITMP3); */
2580                                         /*  #if defined(ENABLE_THREADS) */
2581                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
2582                                         /*  #endif */
2583                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
2584
2585                                         /*                                      } else { */
2586
2587                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, baseval));
2588                                         M_ISUB(REG_ITMP3, REG_ITMP2);
2589                                         M_ALD(REG_ITMP3, RIP, disp);
2590                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2591                                         /*                                      } */
2592
2593                                         M_ICMP(REG_ITMP3, REG_ITMP2);
2594                                         emit_classcast_check(cd, iptr, BRANCH_UGT, REG_ITMP3, s1);
2595
2596                                         if (super != NULL)
2597                                                 emit_label(cd, BRANCH_LABEL_5);
2598                                 }
2599
2600                                 if (super == NULL) {
2601                                         emit_label(cd, BRANCH_LABEL_1);
2602                                         emit_label(cd, BRANCH_LABEL_4);
2603                                 }
2604
2605                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
2606                         }
2607                         else {
2608                                 /* array type cast-check */
2609
2610                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2611                                 M_INTMOVE(s1, REG_A0);
2612
2613                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2614                                         cr   = iptr->sx.s23.s3.c.ref;
2615                                         disp = dseg_add_unique_address(cd, cr);
2616
2617                                         patcher_add_patch_ref(jd,
2618                                                                                   PATCHER_resolve_classref_to_classinfo,
2619                                                                                   cr, disp);
2620                                 }
2621                                 else {
2622                                         disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2623                                 }
2624
2625                                 M_ALD(REG_A1, RIP, disp);
2626                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
2627                                 M_CALL(REG_ITMP1);
2628
2629                                 /* s1 may have been destroyed over the function call */
2630                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2631                                 M_TEST(REG_RESULT);
2632                                 emit_classcast_check(cd, iptr, BRANCH_EQ, REG_RESULT, s1);
2633
2634                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2635                         }
2636
2637                         M_INTMOVE(s1, d);
2638                         emit_store_dst(jd, iptr, d);
2639                         break;
2640
2641                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
2642
2643                         {
2644                         classinfo *super;
2645                         s4         superindex;
2646
2647                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2648                                 super      = NULL;
2649                                 superindex = 0;
2650                         }
2651                         else {
2652                                 super      = iptr->sx.s23.s3.c.cls;
2653                                 superindex = super->index;
2654                         }
2655
2656                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2657                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2658
2659                         if (s1 == d) {
2660                                 M_INTMOVE(s1, REG_ITMP1);
2661                                 s1 = REG_ITMP1;
2662                         }
2663
2664                         M_CLR(d);
2665
2666                         /* if class is not resolved, check which code to call */
2667
2668                         if (super == NULL) {
2669                                 M_TEST(s1);
2670                                 emit_label_beq(cd, BRANCH_LABEL_1);
2671
2672                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_flags,
2673                                                                           iptr->sx.s23.s3.c.ref, 0);
2674
2675                                 M_IMOV_IMM(0, REG_ITMP3);                     /* super->flags */
2676                                 M_IAND_IMM(ACC_INTERFACE, REG_ITMP3);
2677                                 emit_label_beq(cd, BRANCH_LABEL_2);
2678                         }
2679
2680                         /* interface instanceof code */
2681
2682                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2683                                 if (super != NULL) {
2684                                         M_TEST(s1);
2685                                         emit_label_beq(cd, BRANCH_LABEL_3);
2686                                 }
2687
2688                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2689
2690                                 if (super == NULL) {
2691                                         patcher_add_patch_ref(jd, PATCHER_instanceof_interface,
2692                                                                                   iptr->sx.s23.s3.c.ref, 0);
2693                                 }
2694
2695                                 M_ILD32(REG_ITMP3,
2696                                                 REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
2697                                 M_ICMP_IMM32(superindex, REG_ITMP3);
2698
2699                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
2700
2701                                 M_BLE(a);
2702                                 M_ALD32(REG_ITMP1, REG_ITMP1,
2703                                                 OFFSET(vftbl_t, interfacetable[0]) -
2704                                                 superindex * sizeof(methodptr*));
2705                                 M_TEST(REG_ITMP1);
2706                                 M_SETNE(d);
2707
2708                                 if (super == NULL)
2709                                         emit_label_br(cd, BRANCH_LABEL_4);
2710                                 else
2711                                         emit_label(cd, BRANCH_LABEL_3);
2712                         }
2713
2714                         /* class instanceof code */
2715
2716                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2717                                 if (super == NULL) {
2718                                         emit_label(cd, BRANCH_LABEL_2);
2719
2720                                         cr   = iptr->sx.s23.s3.c.ref;
2721                                         disp = dseg_add_unique_address(cd, cr);
2722
2723                                         patcher_add_patch_ref(jd,
2724                                                                                   PATCHER_resolve_classref_to_vftbl,
2725                                                                                   cr, disp);
2726                                 }
2727                                 else {
2728                                         M_TEST(s1);
2729                                         emit_label_beq(cd, BRANCH_LABEL_5);
2730
2731                                         disp = dseg_add_address(cd, super->vftbl);
2732                                 }
2733
2734                                 M_ALD(REG_ITMP1, s1, OFFSET(java_object_t, vftbl));
2735                                 M_ALD(REG_ITMP2, RIP, disp);
2736
2737                                 M_ILD(REG_ITMP1, REG_ITMP1, OFFSET(vftbl_t, baseval));
2738                                 M_ILD(REG_ITMP3, REG_ITMP2, OFFSET(vftbl_t, diffval));
2739                                 M_ILD(REG_ITMP2, REG_ITMP2, OFFSET(vftbl_t, baseval));
2740
2741                                 M_ISUB(REG_ITMP2, REG_ITMP1);
2742                                 M_CLR(d); /* may be REG_ITMP2 */
2743                                 M_ICMP(REG_ITMP3, REG_ITMP1);
2744                                 M_SETULE(d);
2745
2746                                 if (super != NULL)
2747                                         emit_label(cd, BRANCH_LABEL_5);
2748                         }
2749
2750                         if (super == NULL) {
2751                                 emit_label(cd, BRANCH_LABEL_1);
2752                                 emit_label(cd, BRANCH_LABEL_4);
2753                         }
2754
2755                         emit_store_dst(jd, iptr, d);
2756                         }
2757                         break;
2758
2759                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
2760
2761                         /* check for negative sizes and copy sizes to stack if necessary  */
2762
2763                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
2764
2765                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
2766
2767                                 /* copy SAVEDVAR sizes to stack */
2768                                 var = VAR(iptr->sx.s23.s2.args[s1]);
2769
2770                                 /* Already Preallocated? */
2771                                 if (!(var->flags & PREALLOC)) {
2772                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
2773                                         M_LST(s2, REG_SP, s1 * 8);
2774                                 }
2775                         }
2776
2777                         /* a0 = dimension count */
2778
2779                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
2780
2781                         /* is a patcher function set? */
2782
2783                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2784                                 cr   = iptr->sx.s23.s3.c.ref;
2785                                 disp = dseg_add_unique_address(cd, cr);
2786
2787                                 patcher_add_patch_ref(jd, PATCHER_resolve_classref_to_classinfo,
2788                                                                           cr, disp);
2789                         }
2790                         else {
2791                                 disp = dseg_add_address(cd, iptr->sx.s23.s3.c.cls);
2792                         }
2793
2794                         /* a1 = classinfo */
2795
2796                         M_ALD(REG_A1, RIP, disp);
2797
2798                         /* a2 = pointer to dimensions = stack pointer */
2799
2800                         M_MOV(REG_SP, REG_A2);
2801
2802                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
2803                         M_CALL(REG_ITMP1);
2804
2805                         /* check for exception before result assignment */
2806
2807                         emit_exception_check(cd, iptr);
2808
2809                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2810                         M_INTMOVE(REG_RESULT, s1);
2811                         emit_store_dst(jd, iptr, s1);
2812                         break;
2813
2814                 default:
2815                         exceptions_throw_internalerror("Unknown ICMD %d during code generation",
2816                                                                                    iptr->opc);
2817                         return false;
2818         } /* switch */
2819
2820         } /* for instruction */
2821
2822         MCODECHECK(512); /* XXX require a lower number? */
2823
2824         /* At the end of a basic block we may have to append some nops,
2825            because the patcher stub calling code might be longer than the
2826            actual instruction. So codepatching does not change the
2827            following block unintentionally. */
2828
2829         if (cd->mcodeptr < cd->lastmcodeptr) {
2830                 while (cd->mcodeptr < cd->lastmcodeptr) {
2831                         M_NOP;
2832                 }
2833         }
2834
2835         } /* if (bptr -> flags >= BBREACHED) */
2836         } /* for basic block */
2837
2838         /* Generate patcher traps. */
2839
2840         emit_patcher_traps(jd);
2841
2842         /* everything's ok */
2843
2844         return true;
2845 }
2846
2847
2848 /* codegen_emit_stub_native ****************************************************
2849
2850    Emits a stub routine which calls a native method.
2851
2852 *******************************************************************************/
2853
2854 void codegen_emit_stub_native(jitdata *jd, methoddesc *nmd, functionptr f, int skipparams)
2855 {
2856         methodinfo  *m;
2857         codeinfo    *code;
2858         codegendata *cd;
2859         methoddesc  *md;
2860         int          i, j;
2861         int          s1, s2;
2862         int          disp;
2863
2864         /* Sanity check. */
2865
2866         assert(f != NULL);
2867
2868         /* Get required compiler data. */
2869
2870         m    = jd->m;
2871         code = jd->code;
2872         cd   = jd->cd;
2873
2874         /* initialize variables */
2875
2876         md = m->parseddesc;
2877
2878         /* calculate stack frame size */
2879
2880         cd->stackframesize =
2881                 sizeof(stackframeinfo_t) / SIZEOF_VOID_P +
2882                 sizeof(localref_table) / SIZEOF_VOID_P +
2883                 md->paramcount +
2884                 (md->returntype.type == TYPE_VOID ? 0 : 1) +
2885                 nmd->memuse;
2886
2887         ALIGN_ODD(cd->stackframesize);              /* keep stack 16-byte aligned */
2888
2889         /* create method header */
2890
2891         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
2892         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
2893         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
2894         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
2895         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
2896
2897 #if defined(ENABLE_PROFILING)
2898         /* generate native method profiling code */
2899
2900         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
2901                 /* count frequency */
2902
2903                 M_MOV_IMM(code, REG_ITMP3);
2904                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
2905         }
2906 #endif
2907
2908         /* generate stub code */
2909
2910         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
2911
2912 #if defined(ENABLE_GC_CACAO)
2913         /* Save callee saved integer registers in stackframeinfo (GC may
2914            need to recover them during a collection). */
2915
2916         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
2917                 OFFSET(stackframeinfo_t, intregs);
2918
2919         for (i = 0; i < INT_SAV_CNT; i++)
2920                 M_AST(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
2921 #endif
2922
2923         /* save integer and float argument registers */
2924
2925         for (i = 0; i < md->paramcount; i++) {
2926                 if (!md->params[i].inmemory) {
2927                         s1 = md->params[i].regoff;
2928
2929                         switch (md->paramtypes[i].type) {
2930                         case TYPE_INT:
2931                         case TYPE_LNG:
2932                         case TYPE_ADR:
2933                                 M_LST(s1, REG_SP, i * 8);
2934                                 break;
2935                         case TYPE_FLT:
2936                         case TYPE_DBL:
2937                                 M_DST(s1, REG_SP, i * 8);
2938                                 break;
2939                         }
2940                 }
2941         }
2942
2943         /* create dynamic stack info */
2944
2945         M_MOV(REG_SP, REG_A0);
2946         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
2947         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
2948         M_CALL(REG_ITMP1);
2949
2950         /* remember class argument */
2951
2952         if (m->flags & ACC_STATIC)
2953                 M_MOV(REG_RESULT, REG_ITMP2);
2954
2955         /* restore integer and float argument registers */
2956
2957         for (i = 0; i < md->paramcount; i++) {
2958                 if (!md->params[i].inmemory) {
2959                         s1 = md->params[i].regoff;
2960
2961                         switch (md->paramtypes[i].type) {
2962                         case TYPE_INT:
2963                         case TYPE_LNG:
2964                         case TYPE_ADR:
2965                                 M_LLD(s1, REG_SP, i * 8);
2966                                 break;
2967                         case TYPE_FLT:
2968                         case TYPE_DBL:
2969                                 M_DLD(s1, REG_SP, i * 8);
2970                                 break;
2971                         }
2972                 }
2973         }
2974
2975         /* Copy or spill arguments to new locations. */
2976
2977         for (i = md->paramcount - 1, j = i + skipparams; i >= 0; i--, j--) {
2978                 s2 = nmd->params[j].regoff;
2979
2980                 switch (md->paramtypes[i].type) {
2981                 case TYPE_INT:
2982                 case TYPE_LNG:
2983                 case TYPE_ADR:
2984                         if (!md->params[i].inmemory) {
2985                                 s1 = md->params[i].regoff;
2986
2987                                 if (!nmd->params[j].inmemory)
2988                                         M_INTMOVE(s1, s2);
2989                                 else
2990                                         M_LST(s1, REG_SP, s2);
2991                         }
2992                         else {
2993                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
2994                                 M_LLD(REG_ITMP1, REG_SP, s1);
2995                                 M_LST(REG_ITMP1, REG_SP, s2);
2996                         }
2997                         break;
2998                 case TYPE_FLT:
2999                         /* We only copy spilled float arguments, as the float
3000                            argument registers keep unchanged. */
3001
3002                         if (md->params[i].inmemory) {
3003                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3004
3005                                 M_FLD(REG_FTMP1, REG_SP, s1);
3006                                 M_FST(REG_FTMP1, REG_SP, s2);
3007                         }
3008                         break;
3009                 case TYPE_DBL:
3010                         if (md->params[i].inmemory) {
3011                                 s1 = md->params[i].regoff + cd->stackframesize * 8 + 8;/* +1 (RA) */
3012                                 M_DLD(REG_FTMP1, REG_SP, s1);
3013                                 M_DST(REG_FTMP1, REG_SP, s2);
3014                         }
3015                         break;
3016                 }
3017         }
3018
3019         /* Handle native Java methods. */
3020
3021         if (m->flags & ACC_NATIVE) {
3022                 /* put class into second argument register */
3023
3024                 if (m->flags & ACC_STATIC)
3025                         M_MOV(REG_ITMP2, REG_A1);
3026
3027                 /* put env into first argument register */
3028
3029                 M_MOV_IMM(VM_get_jnienv(), REG_A0);
3030         }
3031
3032         /* Call the native function. */
3033
3034         disp = dseg_add_functionptr(cd, f);
3035         M_ALD(REG_ITMP1, RIP, disp);
3036         M_CALL(REG_ITMP1);
3037
3038         /* save return value */
3039
3040         switch (md->returntype.type) {
3041         case TYPE_INT:
3042         case TYPE_LNG:
3043         case TYPE_ADR:
3044                 switch (md->returntype.primitivetype) {
3045                 case PRIMITIVETYPE_BOOLEAN:
3046                         M_BZEXT(REG_RESULT, REG_RESULT);
3047                         break;
3048                 case PRIMITIVETYPE_BYTE:
3049                         M_BSEXT(REG_RESULT, REG_RESULT);
3050                         break;
3051                 case PRIMITIVETYPE_CHAR:
3052                         M_CZEXT(REG_RESULT, REG_RESULT);
3053                         break;
3054                 case PRIMITIVETYPE_SHORT:
3055                         M_SSEXT(REG_RESULT, REG_RESULT);
3056                         break;
3057                 }
3058                 M_LST(REG_RESULT, REG_SP, 0 * 8);
3059                 break;
3060         case TYPE_FLT:
3061         case TYPE_DBL:
3062                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
3063                 break;
3064         case TYPE_VOID:
3065                 break;
3066         }
3067
3068         /* remove native stackframe info */
3069
3070         M_MOV(REG_SP, REG_A0);
3071         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3072         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3073         M_CALL(REG_ITMP1);
3074         M_MOV(REG_RESULT, REG_ITMP3);
3075
3076         /* restore return value */
3077
3078         switch (md->returntype.type) {
3079         case TYPE_INT:
3080         case TYPE_LNG:
3081         case TYPE_ADR:
3082                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
3083                 break;
3084         case TYPE_FLT:
3085         case TYPE_DBL:
3086                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3087                 break;
3088         case TYPE_VOID:
3089                 break;
3090         }
3091
3092 #if defined(ENABLE_GC_CACAO)
3093         /* Restore callee saved integer registers from stackframeinfo (GC
3094            might have modified them during a collection). */
3095
3096         disp = cd->stackframesize * 8 - sizeof(stackframeinfo_t) +
3097                 OFFSET(stackframeinfo_t, intregs);
3098
3099         for (i = 0; i < INT_SAV_CNT; i++)
3100                 M_ALD(abi_registers_integer_saved[i], REG_SP, disp + i * 8);
3101 #endif
3102
3103         /* remove stackframe */
3104
3105         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3106
3107         /* test for exception */
3108
3109         M_TEST(REG_ITMP3);
3110         M_BNE(1);
3111         M_RET;
3112
3113         /* handle exception */
3114
3115         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3116         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3117         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3118
3119         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3120         M_JMP(REG_ITMP3);
3121 }
3122
3123
3124 /*
3125  * These are local overrides for various environment variables in Emacs.
3126  * Please do not remove this and leave it at the end of the file, where
3127  * Emacs will automagically detect them.
3128  * ---------------------------------------------------------------------
3129  * Local variables:
3130  * mode: c
3131  * indent-tabs-mode: t
3132  * c-basic-offset: 4
3133  * tab-width: 4
3134  * End:
3135  * vim:noexpandtab:sw=4:ts=4:
3136  */