src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31             Edwin Steiner
  32
  33    $Id: codegen.c 5830 2006-10-26 11:04:31Z twisti $
  34
  35 */
  36
  37
  38 #include "config.h"
  39
  40 #include <assert.h>
  41 #include <stdio.h>
  42
  43 #include "vm/types.h"
  44
  45 #include "md-abi.h"
  46
  47 #include "vm/jit/x86_64/arch.h"
  48 #include "vm/jit/x86_64/codegen.h"
  49 #include "vm/jit/x86_64/emit.h"
  50
  51 #include "mm/memory.h"
  52 #include "native/jni.h"
  53 #include "native/native.h"
  54
  55 #if defined(ENABLE_THREADS)
  56 # include "threads/native/lock.h"
  57 #endif
  58
  59 #include "vm/builtin.h"
  60 #include "vm/exceptions.h"
  61 #include "vm/global.h"
  62 #include "vm/loader.h"
  63 #include "vm/options.h"
  64 #include "vm/statistics.h"
  65 #include "vm/stringlocal.h"
  66 #include "vm/vm.h"
  67 #include "vm/jit/asmpart.h"
  68 #include "vm/jit/codegen-common.h"
  69 #include "vm/jit/dseg.h"
  70 #include "vm/jit/emit-common.h"
  71 #include "vm/jit/jit.h"
  72 #include "vm/jit/methodheader.h"
  73 #include "vm/jit/parse.h"
  74 #include "vm/jit/patcher.h"
  75 #include "vm/jit/reg.h"
  76 #include "vm/jit/replace.h"
  77
  78 #if defined(ENABLE_LSRA)
  79 # include "vm/jit/allocator/lsra.h"
  80 #endif
  81
  82
  83 /* codegen *********************************************************************
  84
  85    Generates machine code.
  86
  87 *******************************************************************************/
  88
  89 bool codegen(jitdata *jd)
  90 {
  91         methodinfo         *m;
  92         codeinfo           *code;
  93         codegendata        *cd;
  94         registerdata       *rd;
  95         s4                  len, s1, s2, s3, d, disp;
  96         u2                  currentline;
  97         ptrint              a;
  98         s4                  fieldtype;
  99         varinfo            *var, *var1, *var2, *dst;
 100         basicblock         *bptr;
 101         instruction        *iptr;
 102         exception_entry    *ex;
 103         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 104         unresolved_method  *um;
 105         builtintable_entry *bte;
 106         methoddesc         *md;
 107         rplpoint           *replacementpoint;
 108         s4                 varindex;
 109
 110         /* get required compiler data */
 111
 112         m    = jd->m;
 113         code = jd->code;
 114         cd   = jd->cd;
 115         rd   = jd->rd;
 116
 117         /* prevent compiler warnings */
 118
 119         d   = 0;
 120         lm  = NULL;
 121         um  = NULL;
 122         bte = NULL;
 123
 124         {
 125         s4 i, p, t, l;
 126         s4 savedregs_num;
 127
 128         savedregs_num = 0;
 129
 130         /* space to save used callee saved registers */
 131
 132         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 133         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 134
 135         cd->stackframesize = rd->memuse + savedregs_num;
 136
 137 #if defined(ENABLE_THREADS)
 138         /* space to save argument of monitor_enter */
 139
 140         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 141                 cd->stackframesize++;
 142 #endif
 143
 144         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 145            native code e.g. libc or jni (alignment problems with
 146            movaps). */
 147
 148         if (!jd->isleafmethod || opt_verbosecall)
 149                 cd->stackframesize |= 0x1;
 150
 151         /* create method header */
 152
 153         (void) dseg_addaddress(cd, code);                      /* CodeinfoPointer */
 154         (void) dseg_adds4(cd, cd->stackframesize * 8);         /* FrameSize       */
 155
 156 #if defined(ENABLE_THREADS)
 157         /* IsSync contains the offset relative to the stack pointer for the
 158            argument of monitor_exit used in the exception handler. Since the
 159            offset could be zero and give a wrong meaning of the flag it is
 160            offset by one.
 161         */
 162
 163         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 164                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);       /* IsSync          */
 165         else
 166 #endif
 167                 (void) dseg_adds4(cd, 0);                          /* IsSync          */
 168
 169         (void) dseg_adds4(cd, jd->isleafmethod);               /* IsLeaf          */
 170         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave         */
 171         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave         */
 172
 173         (void) dseg_addlinenumbertablesize(cd);
 174
 175         (void) dseg_adds4(cd, jd->exceptiontablelength);       /* ExTableSize     */
 176
 177         /* create exception table */
 178
 179         for (ex = jd->exceptiontable; ex != NULL; ex = ex->down) {
 180                 dseg_addtarget(cd, ex->start);
 181                 dseg_addtarget(cd, ex->end);
 182                 dseg_addtarget(cd, ex->handler);
 183                 (void) dseg_addaddress(cd, ex->catchtype.any);
 184         }
 185
 186         /* generate method profiling code */
 187
 188         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 189                 /* count frequency */
 190
 191                 M_MOV_IMM(code, REG_ITMP3);
 192                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 193
 194                 PROFILE_CYCLE_START;
 195         }
 196
 197         /* create stack frame (if necessary) */
 198
 199         if (cd->stackframesize)
 200                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 201
 202         /* save used callee saved registers */
 203
 204         p = cd->stackframesize;
 205         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 206                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 207         }
 208         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 209                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 210         }
 211
 212         /* take arguments out of register or stack frame */
 213
 214         md = m->parseddesc;
 215
 216         for (p = 0, l = 0; p < md->paramcount; p++) {
 217                 t = md->paramtypes[p].type;
 218
 219                 varindex = jd->local_map[l * 5 + t];
 220
 221                 l++;
 222                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 223                         l++;
 224
 225                 if (varindex == UNUSED)
 226                         continue;
 227
 228                 var = VAR(varindex);
 229
 230                 s1 = md->params[p].regoff;
 231
 232                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 233                         s2 = rd->argintregs[s1];
 234                         if (!md->params[p].inmemory) {           /* register arguments    */
 235                                 if (!IS_INMEMORY(var->flags)) {      /* reg arg -> register   */
 236                                         M_INTMOVE(s2, var->vv.regoff);
 237
 238                                 } else {                             /* reg arg -> spilled    */
 239                                     M_LST(s2, REG_SP, var->vv.regoff * 8);
 240                                 }
 241
 242                         } else {                                 /* stack arguments       */
 243                                 if (!IS_INMEMORY(var->flags)) {      /* stack arg -> register */
 244                                         /* + 8 for return address */
 245                                         M_LLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 246
 247                                 } else {                             /* stack arg -> spilled  */
 248                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 249                                 }
 250                         }
 251
 252                 } else {                                     /* floating args         */
 253                         if (!md->params[p].inmemory) {           /* register arguments    */
 254                                 s2 = rd->argfltregs[s1];
 255                                 if (!IS_INMEMORY(var->flags)) {      /* reg arg -> register   */
 256                                         M_FLTMOVE(s2, var->vv.regoff);
 257
 258                                 } else {                                         /* reg arg -> spilled    */
 259                                         M_DST(s2, REG_SP, var->vv.regoff * 8);
 260                                 }
 261
 262                         } else {                                 /* stack arguments       */
 263                                 if (!IS_INMEMORY(var->flags)) {      /* stack-arg -> register */
 264                                         M_DLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 265
 266                                 } else {
 267                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 268                                 }
 269                         }
 270                 }
 271         }  /* end for */
 272
 273         /* save monitorenter argument */
 274
 275 #if defined(ENABLE_THREADS)
 276         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 277                 /* stack offset for monitor argument */
 278
 279                 s1 = rd->memuse;
 280
 281                 if (opt_verbosecall) {
 282                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 283
 284                         for (p = 0; p < INT_ARG_CNT; p++)
 285                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 286
 287                         for (p = 0; p < FLT_ARG_CNT; p++)
 288                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 289
 290                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 291                 }
 292
 293                 /* decide which monitor enter function to call */
 294
 295                 if (m->flags & ACC_STATIC) {
 296                         M_MOV_IMM(&m->class->object.header, REG_A0);
 297                 }
 298                 else {
 299                         M_TEST(REG_A0);
 300                         M_BEQ(0);
 301                         codegen_add_nullpointerexception_ref(cd);
 302                 }
 303
 304                 M_AST(REG_A0, REG_SP, s1 * 8);
 305                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 306                 M_CALL(REG_ITMP1);
 307
 308                 if (opt_verbosecall) {
 309                         for (p = 0; p < INT_ARG_CNT; p++)
 310                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 311
 312                         for (p = 0; p < FLT_ARG_CNT; p++)
 313                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 314
 315                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 316                 }
 317         }
 318 #endif
 319
 320 #if !defined(NDEBUG)
 321         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 322                 emit_verbosecall_enter(jd);
 323 #endif /* !defined(NDEBUG) */
 324
 325         }
 326
 327         /* end of header generation */
 328
 329         replacementpoint = jd->code->rplpoints;
 330
 331         /* walk through all basic blocks */
 332
 333         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 334
 335                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 336
 337                 if (bptr->flags >= BBREACHED) {
 338
 339                         /* branch resolving */
 340
 341                         branchref *bref;
 342                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 343                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 344                                                                   bref->branchpos,
 345                                                                   bptr->mpc);
 346                         }
 347
 348                 /* handle replacement points */
 349
 350 #if 0
 351                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 352                         replacementpoint->pc = (u1*)(ptrint)bptr->mpc; /* will be resolved later */
 353
 354                         replacementpoint++;
 355
 356                         assert(cd->lastmcodeptr <= cd->mcodeptr);
 357                         cd->lastmcodeptr = cd->mcodeptr + 5; /* 5 byte jmp patch */
 358                 }
 359 #endif
 360
 361                 /* copy interface registers to their destination */
 362
 363                 len = bptr->indepth;
 364                 MCODECHECK(512);
 365
 366                 /* generate basicblock profiling code */
 367
 368                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 369                         /* count frequency */
 370
 371                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 372                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 373
 374                         /* if this is an exception handler, start profiling again */
 375
 376                         if (bptr->type == BBTYPE_EXH)
 377                                 PROFILE_CYCLE_START;
 378                 }
 379
 380 #if defined(ENABLE_LSRA)
 381                 if (opt_lsra) {
 382                         while (len) {
 383                                 len--;
 384                                 src = bptr->invars[len];
 385                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 386                                         if (bptr->type == BBTYPE_EXH) {
 387 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 388                                                 if (!IS_INMEMORY(src->flags))
 389                                                         d= src->vv.regoff;
 390                                                 else
 391                                                         d=REG_ITMP1;
 392                                                 M_INTMOVE(REG_ITMP1, d);
 393                                                 emit_store(jd, NULL, src, d);
 394                                         }
 395                                 }
 396                         }
 397
 398                 } else {
 399 #endif
 400
 401                 while (len) {
 402                         len--;
 403                         var = VAR(bptr->invars[len]);
 404                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 405                                 if (bptr->type == BBTYPE_EXH) {
 406                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 407                                         M_INTMOVE(REG_ITMP1, d);
 408                                         emit_store(jd, NULL, var, d);
 409                                 }
 410                         }
 411                         else {
 412                                 assert((var->flags & INOUT));
 413                         }
 414                 }
 415 #if defined(ENABLE_LSRA)
 416                 }
 417 #endif
 418                 /* walk through all instructions */
 419
 420                 len = bptr->icount;
 421                 currentline = 0;
 422
 423                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 424                         if (iptr->line != currentline) {
 425                                 dseg_addlinenumber(cd, iptr->line);
 426                                 currentline = iptr->line;
 427                         }
 428
 429                         MCODECHECK(1024);                         /* 1KB should be enough */
 430
 431                         switch (iptr->opc) {
 432                         case ICMD_INLINE_START: /* internal ICMDs                         */
 433                         case ICMD_INLINE_END:
 434                                 break;
 435
 436                         case ICMD_NOP:    /* ...  ==> ...                                 */
 437                                 break;
 438
 439                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 440
 441                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 442                                 M_TEST(s1);
 443                                 M_BEQ(0);
 444                                 codegen_add_nullpointerexception_ref(cd);
 445                                 break;
 446
 447                 /* constant operations ************************************************/
 448
 449                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 450
 451                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 452                         ICONST(d, iptr->sx.val.i);
 453                         emit_store_dst(jd, iptr, d);
 454                         break;
 455
 456                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 457
 458                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 459                         LCONST(d, iptr->sx.val.l);
 460                         emit_store_dst(jd, iptr, d);
 461                         break;
 462
 463                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 464
 465                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 466                         disp = dseg_addfloat(cd, iptr->sx.val.f);
 467                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 468                         emit_store_dst(jd, iptr, d);
 469                         break;
 470
 471                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 472
 473                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 474                         disp = dseg_adddouble(cd, iptr->sx.val.d);
 475                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 476                         emit_store_dst(jd, iptr, d);
 477                         break;
 478
 479                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 480
 481                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 482
 483                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 484 /*                              PROFILE_CYCLE_STOP; */
 485
 486                                 codegen_addpatchref(cd, PATCHER_aconst,
 487                                                                         iptr->sx.val.c.ref, 0);
 488
 489                                 if (opt_showdisassemble) {
 490                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
 491                                 }
 492
 493 /*                              PROFILE_CYCLE_START; */
 494
 495                                 M_MOV_IMM(NULL, d);
 496
 497                         } else {
 498                                 if (iptr->sx.val.anyptr == 0)
 499                                         M_CLR(d);
 500                                 else
 501                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 502                         }
 503                         emit_store_dst(jd, iptr, d);
 504                         break;
 505
 506
 507                 /* load/store/copy/move operations ************************************/
 508
 509                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 510                 case ICMD_ALOAD:      /* s1 = local variable                          */
 511                 case ICMD_LLOAD:
 512                 case ICMD_FLOAD:
 513                 case ICMD_DLOAD:
 514                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 515                 case ICMD_ASTORE:     /* dst = local variable                         */
 516                 case ICMD_LSTORE:
 517                 case ICMD_FSTORE:
 518                 case ICMD_DSTORE:
 519                 case ICMD_COPY:
 520                 case ICMD_MOVE:
 521
 522                         emit_copy(jd, iptr, VAROP(iptr->s1), VAROP(iptr->dst));
 523                         break;
 524
 525
 526                 /* pop operations *****************************************************/
 527
 528                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 529
 530                 case ICMD_POP:        /* ..., value  ==> ...                          */
 531                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 532
 533                         break;
 534
 535
 536                 /* integer operations *************************************************/
 537
 538                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 539
 540                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 541                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 542                         M_INTMOVE(s1, d);
 543                         M_INEG(d);
 544                         emit_store_dst(jd, iptr, d);
 545                         break;
 546
 547                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 548
 549                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 550                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 551                         M_INTMOVE(s1, d);
 552                         M_LNEG(d);
 553                         emit_store_dst(jd, iptr, d);
 554                         break;
 555
 556                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 557
 558                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 559                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 560                         M_ISEXT(s1, d);
 561                         emit_store_dst(jd, iptr, d);
 562                         break;
 563
 564                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 565
 566                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 567                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 568                         M_IMOV(s1, d);
 569                         emit_store_dst(jd, iptr, d);
 570                         break;
 571
 572                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 573
 574                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 575                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 576                         M_BSEXT(s1, d);
 577                         emit_store_dst(jd, iptr, d);
 578                         break;
 579
 580                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 581
 582                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 583                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 584                         M_CZEXT(s1, d);
 585                         emit_store_dst(jd, iptr, d);
 586                         break;
 587
 588                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 589
 590                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 591                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 592                         M_SSEXT(s1, d);
 593                         emit_store_dst(jd, iptr, d);
 594                         break;
 595
 596
 597                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 598
 599                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 600                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 601                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 602                         if (s2 == d)
 603                                 M_IADD(s1, d);
 604                         else {
 605                                 M_INTMOVE(s1, d);
 606                                 M_IADD(s2, d);
 607                         }
 608                         emit_store_dst(jd, iptr, d);
 609                         break;
 610
 611                 case ICMD_IINC:
 612                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 613                                       /* sx.val.i = constant                             */
 614
 615                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 616                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 617
 618                         /* Using inc and dec is not faster than add (tested with
 619                            sieve). */
 620
 621                         M_INTMOVE(s1, d);
 622                         M_IADD_IMM(iptr->sx.val.i, d);
 623                         emit_store_dst(jd, iptr, d);
 624                         break;
 625
 626                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 627
 628                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 629                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 630                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 631                         if (s2 == d)
 632                                 M_LADD(s1, d);
 633                         else {
 634                                 M_INTMOVE(s1, d);
 635                                 M_LADD(s2, d);
 636                         }
 637                         emit_store_dst(jd, iptr, d);
 638                         break;
 639
 640                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 641                                       /* sx.val.l = constant                             */
 642
 643                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 644                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 645                         M_INTMOVE(s1, d);
 646                         if (IS_IMM32(iptr->sx.val.l))
 647                                 M_LADD_IMM(iptr->sx.val.l, d);
 648                         else {
 649                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 650                                 M_LADD(REG_ITMP2, d);
 651                         }
 652                         emit_store_dst(jd, iptr, d);
 653                         break;
 654
 655                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 656
 657                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 658                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 659                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 660                         if (s2 == d) {
 661                                 M_INTMOVE(s1, REG_ITMP1);
 662                                 M_ISUB(s2, REG_ITMP1);
 663                                 M_INTMOVE(REG_ITMP1, d);
 664                         } else {
 665                                 M_INTMOVE(s1, d);
 666                                 M_ISUB(s2, d);
 667                         }
 668                         emit_store_dst(jd, iptr, d);
 669                         break;
 670
 671                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 672                                       /* sx.val.i = constant                             */
 673
 674                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 675                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 676                         M_INTMOVE(s1, d);
 677                         M_ISUB_IMM(iptr->sx.val.i, d);
 678                         emit_store_dst(jd, iptr, d);
 679                         break;
 680
 681                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 682
 683                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 684                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 685                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 686                         if (s2 == d) {
 687                                 M_INTMOVE(s1, REG_ITMP1);
 688                                 M_LSUB(s2, REG_ITMP1);
 689                                 M_INTMOVE(REG_ITMP1, d);
 690                         } else {
 691                                 M_INTMOVE(s1, d);
 692                                 M_LSUB(s2, d);
 693                         }
 694                         emit_store_dst(jd, iptr, d);
 695                         break;
 696
 697                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 698                                       /* sx.val.l = constant                             */
 699
 700                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 701                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 702                         M_INTMOVE(s1, d);
 703                         if (IS_IMM32(iptr->sx.val.l))
 704                                 M_LSUB_IMM(iptr->sx.val.l, d);
 705                         else {
 706                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 707                                 M_LSUB(REG_ITMP2, d);
 708                         }
 709                         emit_store_dst(jd, iptr, d);
 710                         break;
 711
 712                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 713
 714                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 715                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 716                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 717                         if (s2 == d)
 718                                 M_IMUL(s1, d);
 719                         else {
 720                                 M_INTMOVE(s1, d);
 721                                 M_IMUL(s2, d);
 722                         }
 723                         emit_store_dst(jd, iptr, d);
 724                         break;
 725
 726                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 727                                       /* sx.val.i = constant                             */
 728
 729                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 730                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 731                         if (iptr->sx.val.i == 2) {
 732                                 M_INTMOVE(s1, d);
 733                                 M_ISLL_IMM(1, d);
 734                         } else
 735                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 736                         emit_store_dst(jd, iptr, d);
 737                         break;
 738
 739                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 740
 741                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 742                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 743                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 744                         if (s2 == d)
 745                                 M_LMUL(s1, d);
 746                         else {
 747                                 M_INTMOVE(s1, d);
 748                                 M_LMUL(s2, d);
 749                         }
 750                         emit_store_dst(jd, iptr, d);
 751                         break;
 752
 753                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 754                                       /* sx.val.l = constant                             */
 755
 756                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 757                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 758                         if (IS_IMM32(iptr->sx.val.l))
 759                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 760                         else {
 761                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 762                                 M_INTMOVE(s1, d);
 763                                 M_LMUL(REG_ITMP2, d);
 764                         }
 765                         emit_store_dst(jd, iptr, d);
 766                         break;
 767
 768                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 769
 770                         var1 = VAROP(iptr->s1);
 771                         var2 = VAROP(iptr->sx.s23.s2);
 772                         dst  = VAROP(iptr->dst);
 773
 774                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 775                         if (IS_INMEMORY(var1->flags))
 776                                 M_ILD(RAX, REG_SP, var1->vv.regoff * 8);
 777                         else
 778                                 M_INTMOVE(var1->vv.regoff, RAX);
 779
 780                         if (IS_INMEMORY(var2->flags))
 781                                 M_ILD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 782                         else
 783                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 784
 785                         if (checknull) {
 786                                 M_ITEST(REG_ITMP3);
 787                                 M_BEQ(0);
 788                                 codegen_add_arithmeticexception_ref(cd);
 789                         }
 790
 791                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
 792                         emit_jcc(cd, CC_NE, 4 + 6);
 793                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
 794                         emit_jcc(cd, CC_E, 3 + 1 + 3);                /* 6 bytes */
 795
 796                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 797                         emit_cltd(cd);
 798                         emit_idivl_reg(cd, REG_ITMP3);
 799
 800                         if (IS_INMEMORY(dst->flags)) {
 801                                 emit_mov_reg_membase(cd, RAX, REG_SP, dst->vv.regoff * 8);
 802                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
 803
 804                         } else {
 805                                 M_INTMOVE(RAX, dst->vv.regoff);
 806
 807                                 if (dst->vv.regoff != RDX) {
 808                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
 809                                 }
 810                         }
 811                         break;
 812
 813                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 814
 815                         var1 = VAROP(iptr->s1);
 816                         var2 = VAROP(iptr->sx.s23.s2);
 817                         dst  = VAROP(iptr->dst);
 818
 819                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 820                         if (IS_INMEMORY(var1->flags))
 821                                 M_ILD(RAX, REG_SP, var1->vv.regoff * 8);
 822                         else
 823                                 M_INTMOVE(var1->vv.regoff, RAX);
 824
 825                         if (IS_INMEMORY(var2->flags))
 826                                 M_ILD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 827                         else
 828                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 829
 830                         if (checknull) {
 831                                 M_ITEST(REG_ITMP3);
 832                                 M_BEQ(0);
 833                                 codegen_add_arithmeticexception_ref(cd);
 834                         }
 835
 836                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 837
 838                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
 839                         emit_jcc(cd, CC_NE, 2 + 4 + 6);
 840
 841
 842                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
 843                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
 844                         emit_jcc(cd, CC_E, 1 + 3);                    /* 6 bytes */
 845
 846                         emit_cltd(cd);
 847                         emit_idivl_reg(cd, REG_ITMP3);
 848
 849                         if (IS_INMEMORY(dst->flags)) {
 850                                 emit_mov_reg_membase(cd, RDX, REG_SP, dst->vv.regoff * 8);
 851                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
 852
 853                         } else {
 854                                 M_INTMOVE(RDX, dst->vv.regoff);
 855
 856                                 if (dst->vv.regoff != RDX) {
 857                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
 858                                 }
 859                         }
 860                         break;
 861
 862                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 863                                       /* sx.val.i = constant                             */
 864
 865                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 866                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 867                         M_INTMOVE(s1, REG_ITMP1);
 868                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 869                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 870                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 871                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 872                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 873                         emit_store_dst(jd, iptr, d);
 874                         break;
 875
 876                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 877                                       /* sx.val.i = constant                             */
 878
 879                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 880                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 881                         M_INTMOVE(s1, REG_ITMP1);
 882                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 883                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 884                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 885                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 886                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 887                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 888                         emit_store_dst(jd, iptr, d);
 889                         break;
 890
 891
 892                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 893
 894                         var1 = VAROP(iptr->s1);
 895                         var2 = VAROP(iptr->sx.s23.s2);
 896                         dst  = VAROP(iptr->dst);
 897
 898                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 899
 900                 if (IS_INMEMORY(var1->flags))
 901                                 M_LLD(RAX, REG_SP, var1->vv.regoff * 8);
 902                         else
 903                                 M_INTMOVE(var1->vv.regoff, RAX);
 904
 905                         if (IS_INMEMORY(var2->flags))
 906                                 M_LLD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 907                         else
 908                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 909
 910                         if (checknull) {
 911                                 M_TEST(REG_ITMP3);
 912                                 M_BEQ(0);
 913                                 codegen_add_arithmeticexception_ref(cd);
 914                         }
 915
 916                         /* check as described in jvm spec */
 917                         disp = dseg_adds8(cd, 0x8000000000000000LL);
 918                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 919                         M_BNE(4 + 6);
 920                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
 921                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
 922
 923                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 924                         emit_cqto(cd);
 925                         emit_idiv_reg(cd, REG_ITMP3);
 926
 927                         if (IS_INMEMORY(dst->flags)) {
 928                                 M_LST(RAX, REG_SP, dst->vv.regoff * 8);
 929                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
 930
 931                         } else {
 932                                 M_INTMOVE(RAX, dst->vv.regoff);
 933
 934                                 if (dst->vv.regoff != RDX) {
 935                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
 936                                 }
 937                         }
 938                         break;
 939
 940                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 941
 942                         var1 = VAROP(iptr->s1);
 943                         var2 = VAROP(iptr->sx.s23.s2);
 944                         dst  = VAROP(iptr->dst);
 945
 946                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 947
 948                         if (IS_INMEMORY(var1->flags))
 949                                 M_LLD(REG_ITMP1, REG_SP, var1->vv.regoff * 8);
 950                         else
 951                                 M_INTMOVE(var1->vv.regoff, REG_ITMP1);
 952
 953                         if (IS_INMEMORY(var2->flags))
 954                                 M_LLD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 955                         else
 956                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 957                         /*
 958                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 959                         M_INTMOVE(s1, REG_ITMP1);
 960                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 961                         M_INTMOVE(s2, REG_ITMP3);
 962                         */
 963                         if (checknull) {
 964                                 M_ITEST(REG_ITMP3);
 965                                 M_BEQ(0);
 966                                 codegen_add_arithmeticexception_ref(cd);
 967                         }
 968
 969                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 970
 971                         /* check as described in jvm spec */
 972                         disp = dseg_adds8(cd, 0x8000000000000000LL);
 973                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 974                         M_BNE(3 + 4 + 6);
 975
 976 #if 0
 977                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
 978 #endif
 979                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 980                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
 981                         M_BEQ(2 + 3);                                          /* 6 bytes */
 982
 983                         emit_cqto(cd);
 984                         emit_idiv_reg(cd, REG_ITMP3);
 985
 986                         if (IS_INMEMORY(dst->flags)) {
 987                                 M_LST(RDX, REG_SP, dst->vv.regoff * 8);
 988                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
 989
 990                         } else {
 991                                 M_INTMOVE(RDX, dst->vv.regoff);
 992
 993                                 if (dst->vv.regoff != RDX) {
 994                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
 995                                 }
 996                         }
 997                         break;
 998
 999                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1000                                       /* sx.val.i = constant                             */
1001
1002                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1003                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1004                         M_INTMOVE(s1, REG_ITMP1);
1005                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1006                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
1007                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
1008                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
1009                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1010                         emit_store_dst(jd, iptr, d);
1011                         break;
1012
1013                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1014                                       /* sx.val.l = constant                             */
1015
1016                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1017                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1018                         M_INTMOVE(s1, REG_ITMP1);
1019                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1020                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
1021                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
1022                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
1023                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
1024                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1025                         emit_store_dst(jd, iptr, d);
1026                         break;
1027
1028                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1029
1030                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1031                         emit_ishift(jd, SHIFT_SHL, iptr);
1032                         break;
1033
1034                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1035                                       /* sx.val.i = constant                             */
1036
1037                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1038                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1039                         M_INTMOVE(s1, d);
1040                         M_ISLL_IMM(iptr->sx.val.i, d);
1041                         emit_store_dst(jd, iptr, d);
1042                         break;
1043
1044                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1045
1046                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1047                         emit_ishift(jd, SHIFT_SAR, iptr);
1048                         break;
1049
1050                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1051                                       /* sx.val.i = constant                             */
1052
1053                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1054                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1055                         M_INTMOVE(s1, d);
1056                         M_ISRA_IMM(iptr->sx.val.i, d);
1057                         emit_store_dst(jd, iptr, d);
1058                         break;
1059
1060                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1061
1062                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1063                         emit_ishift(jd, SHIFT_SHR, iptr);
1064                         break;
1065
1066                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1067                                       /* sx.val.i = constant                             */
1068
1069                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1070                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1071                         M_INTMOVE(s1, d);
1072                         M_ISRL_IMM(iptr->sx.val.i, d);
1073                         emit_store_dst(jd, iptr, d);
1074                         break;
1075
1076                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1077
1078                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1079                         emit_lshift(jd, SHIFT_SHL, iptr);
1080                         break;
1081
1082         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1083                                           /* sx.val.i = constant                             */
1084
1085                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1086                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1087                         M_INTMOVE(s1, d);
1088                         M_LSLL_IMM(iptr->sx.val.i, d);
1089                         emit_store_dst(jd, iptr, d);
1090                         break;
1091
1092                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1093
1094                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1095                         emit_lshift(jd, SHIFT_SAR, iptr);
1096                         break;
1097
1098                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1099                                       /* sx.val.i = constant                             */
1100
1101                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1102                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1103                         M_INTMOVE(s1, d);
1104                         M_LSRA_IMM(iptr->sx.val.i, d);
1105                         emit_store_dst(jd, iptr, d);
1106                         break;
1107
1108                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1109
1110                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1111                         emit_lshift(jd, SHIFT_SHR, iptr);
1112                         break;
1113
1114                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1115                                       /* sx.val.l = constant                             */
1116
1117                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1118                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1119                         M_INTMOVE(s1, d);
1120                         M_LSRL_IMM(iptr->sx.val.i, d);
1121                         emit_store_dst(jd, iptr, d);
1122                         break;
1123
1124                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1125
1126                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1127                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1128                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1129                         if (s2 == d)
1130                                 M_IAND(s1, d);
1131                         else {
1132                                 M_INTMOVE(s1, d);
1133                                 M_IAND(s2, d);
1134                         }
1135                         emit_store_dst(jd, iptr, d);
1136                         break;
1137
1138                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1139                                       /* sx.val.i = constant                             */
1140
1141                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1142                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1143                         M_INTMOVE(s1, d);
1144                         M_IAND_IMM(iptr->sx.val.i, d);
1145                         emit_store_dst(jd, iptr, d);
1146                         break;
1147
1148                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1149
1150                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1151                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1152                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1153                         if (s2 == d)
1154                                 M_LAND(s1, d);
1155                         else {
1156                                 M_INTMOVE(s1, d);
1157                                 M_LAND(s2, d);
1158                         }
1159                         emit_store_dst(jd, iptr, d);
1160                         break;
1161
1162                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1163                                       /* sx.val.l = constant                             */
1164
1165                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1166                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1167                         M_INTMOVE(s1, d);
1168                         if (IS_IMM32(iptr->sx.val.l))
1169                                 M_LAND_IMM(iptr->sx.val.l, d);
1170                         else {
1171                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1172                                 M_LAND(REG_ITMP2, d);
1173                         }
1174                         emit_store_dst(jd, iptr, d);
1175                         break;
1176
1177                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1178
1179                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1180                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1181                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1182                         if (s2 == d)
1183                                 M_IOR(s1, d);
1184                         else {
1185                                 M_INTMOVE(s1, d);
1186                                 M_IOR(s2, d);
1187                         }
1188                         emit_store_dst(jd, iptr, d);
1189                         break;
1190
1191                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1192                                       /* sx.val.i = constant                             */
1193
1194                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1195                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1196                         M_INTMOVE(s1, d);
1197                         M_IOR_IMM(iptr->sx.val.i, d);
1198                         emit_store_dst(jd, iptr, d);
1199                         break;
1200
1201                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1202
1203                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1204                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1205                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1206                         if (s2 == d)
1207                                 M_LOR(s1, d);
1208                         else {
1209                                 M_INTMOVE(s1, d);
1210                                 M_LOR(s2, d);
1211                         }
1212                         emit_store_dst(jd, iptr, d);
1213                         break;
1214
1215                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1216                                       /* sx.val.l = constant                             */
1217
1218                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1219                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1220                         M_INTMOVE(s1, d);
1221                         if (IS_IMM32(iptr->sx.val.l))
1222                                 M_LOR_IMM(iptr->sx.val.l, d);
1223                         else {
1224                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1225                                 M_LOR(REG_ITMP2, d);
1226                         }
1227                         emit_store_dst(jd, iptr, d);
1228                         break;
1229
1230                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1231
1232                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1233                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1234                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1235                         if (s2 == d)
1236                                 M_IXOR(s1, d);
1237                         else {
1238                                 M_INTMOVE(s1, d);
1239                                 M_IXOR(s2, d);
1240                         }
1241                         emit_store_dst(jd, iptr, d);
1242                         break;
1243
1244                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1245                                       /* sx.val.i = constant                             */
1246
1247                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1248                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1249                         M_INTMOVE(s1, d);
1250                         M_IXOR_IMM(iptr->sx.val.i, d);
1251                         emit_store_dst(jd, iptr, d);
1252                         break;
1253
1254                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1255
1256                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1257                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1258                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1259                         if (s2 == d)
1260                                 M_LXOR(s1, d);
1261                         else {
1262                                 M_INTMOVE(s1, d);
1263                                 M_LXOR(s2, d);
1264                         }
1265                         emit_store_dst(jd, iptr, d);
1266                         break;
1267
1268                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1269                                       /* sx.val.l = constant                             */
1270
1271                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1272                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1273                         M_INTMOVE(s1, d);
1274                         if (IS_IMM32(iptr->sx.val.l))
1275                                 M_LXOR_IMM(iptr->sx.val.l, d);
1276                         else {
1277                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1278                                 M_LXOR(REG_ITMP2, d);
1279                         }
1280                         emit_store_dst(jd, iptr, d);
1281                         break;
1282
1283
1284                 /* floating operations ************************************************/
1285
1286                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1287
1288                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1289                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1290                         disp = dseg_adds4(cd, 0x80000000);
1291                         M_FLTMOVE(s1, d);
1292                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1293                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1294                         emit_store_dst(jd, iptr, d);
1295                         break;
1296
1297                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1298
1299                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1300                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1301                         disp = dseg_adds8(cd, 0x8000000000000000);
1302                         M_FLTMOVE(s1, d);
1303                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1304                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1305                         emit_store_dst(jd, iptr, d);
1306                         break;
1307
1308                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1309
1310                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1311                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1312                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1313                         if (s2 == d)
1314                                 M_FADD(s1, d);
1315                         else {
1316                                 M_FLTMOVE(s1, d);
1317                                 M_FADD(s2, d);
1318                         }
1319                         emit_store_dst(jd, iptr, d);
1320                         break;
1321
1322                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1323
1324                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1325                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1326                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1327                         if (s2 == d)
1328                                 M_DADD(s1, d);
1329                         else {
1330                                 M_FLTMOVE(s1, d);
1331                                 M_DADD(s2, d);
1332                         }
1333                         emit_store_dst(jd, iptr, d);
1334                         break;
1335
1336                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1337
1338                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1339                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1340                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1341                         if (s2 == d) {
1342                                 M_FLTMOVE(s2, REG_FTMP2);
1343                                 s2 = REG_FTMP2;
1344                         }
1345                         M_FLTMOVE(s1, d);
1346                         M_FSUB(s2, d);
1347                         emit_store_dst(jd, iptr, d);
1348                         break;
1349
1350                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1351
1352                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1353                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1354                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1355                         if (s2 == d) {
1356                                 M_FLTMOVE(s2, REG_FTMP2);
1357                                 s2 = REG_FTMP2;
1358                         }
1359                         M_FLTMOVE(s1, d);
1360                         M_DSUB(s2, d);
1361                         emit_store_dst(jd, iptr, d);
1362                         break;
1363
1364                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1365
1366                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1367                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1368                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1369                         if (s2 == d)
1370                                 M_FMUL(s1, d);
1371                         else {
1372                                 M_FLTMOVE(s1, d);
1373                                 M_FMUL(s2, d);
1374                         }
1375                         emit_store_dst(jd, iptr, d);
1376                         break;
1377
1378                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1379
1380                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1381                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1382                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1383                         if (s2 == d)
1384                                 M_DMUL(s1, d);
1385                         else {
1386                                 M_FLTMOVE(s1, d);
1387                                 M_DMUL(s2, d);
1388                         }
1389                         emit_store_dst(jd, iptr, d);
1390                         break;
1391
1392                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1393
1394                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1395                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1396                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1397                         if (s2 == d) {
1398                                 M_FLTMOVE(s2, REG_FTMP2);
1399                                 s2 = REG_FTMP2;
1400                         }
1401                         M_FLTMOVE(s1, d);
1402                         M_FDIV(s2, d);
1403                         emit_store_dst(jd, iptr, d);
1404                         break;
1405
1406                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1407
1408                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1409                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1410                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1411                         if (s2 == d) {
1412                                 M_FLTMOVE(s2, REG_FTMP2);
1413                                 s2 = REG_FTMP2;
1414                         }
1415                         M_FLTMOVE(s1, d);
1416                         M_DDIV(s2, d);
1417                         emit_store_dst(jd, iptr, d);
1418                         break;
1419
1420                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1421
1422                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1423                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1424                         M_CVTIF(s1, d);
1425                         emit_store_dst(jd, iptr, d);
1426                         break;
1427
1428                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1429
1430                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1431                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1432                         M_CVTID(s1, d);
1433                         emit_store_dst(jd, iptr, d);
1434                         break;
1435
1436                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1437
1438                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1439                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1440                         M_CVTLF(s1, d);
1441                         emit_store_dst(jd, iptr, d);
1442                         break;
1443
1444                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1445
1446                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1447                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1448                         M_CVTLD(s1, d);
1449                         emit_store_dst(jd, iptr, d);
1450                         break;
1451
1452                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1453
1454                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1455                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1456                         M_CVTFI(s1, d);
1457                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1458                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1459                                 ((REG_RESULT == d) ? 0 : 3);
1460                         M_BNE(disp);
1461                         M_FLTMOVE(s1, REG_FTMP1);
1462                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1463                         M_CALL(REG_ITMP2);
1464                         M_INTMOVE(REG_RESULT, d);
1465                         emit_store_dst(jd, iptr, d);
1466                         break;
1467
1468                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1469
1470                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1471                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1472                         M_CVTDI(s1, d);
1473                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1474                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1475                                 ((REG_RESULT == d) ? 0 : 3);
1476                         M_BNE(disp);
1477                         M_FLTMOVE(s1, REG_FTMP1);
1478                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1479                         M_CALL(REG_ITMP2);
1480                         M_INTMOVE(REG_RESULT, d);
1481                         emit_store_dst(jd, iptr, d);
1482                         break;
1483
1484                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1485
1486                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1487                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1488                         M_CVTFL(s1, d);
1489                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1490                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1491                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1492                                 ((REG_RESULT == d) ? 0 : 3);
1493                         M_BNE(disp);
1494                         M_FLTMOVE(s1, REG_FTMP1);
1495                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1496                         M_CALL(REG_ITMP2);
1497                         M_INTMOVE(REG_RESULT, d);
1498                         emit_store_dst(jd, iptr, d);
1499                         break;
1500
1501                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1502
1503                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1504                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1505                         M_CVTDL(s1, d);
1506                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1507                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1508                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1509                                 ((REG_RESULT == d) ? 0 : 3);
1510                         M_BNE(disp);
1511                         M_FLTMOVE(s1, REG_FTMP1);
1512                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1513                         M_CALL(REG_ITMP2);
1514                         M_INTMOVE(REG_RESULT, d);
1515                         emit_store_dst(jd, iptr, d);
1516                         break;
1517
1518                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1519
1520                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1521                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1522                         M_CVTFD(s1, d);
1523                         emit_store_dst(jd, iptr, d);
1524                         break;
1525
1526                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1527
1528                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1529                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1530                         M_CVTDF(s1, d);
1531                         emit_store_dst(jd, iptr, d);
1532                         break;
1533
1534                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1535                                           /* == => 0, < => 1, > => -1 */
1536
1537                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1538                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1539                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1540                         M_CLR(d);
1541                         M_MOV_IMM(1, REG_ITMP1);
1542                         M_MOV_IMM(-1, REG_ITMP2);
1543                         emit_ucomiss_reg_reg(cd, s1, s2);
1544                         M_CMOVB(REG_ITMP1, d);
1545                         M_CMOVA(REG_ITMP2, d);
1546                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1547                         emit_store_dst(jd, iptr, d);
1548                         break;
1549
1550                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1551                                           /* == => 0, < => 1, > => -1 */
1552
1553                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1554                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1555                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1556                         M_CLR(d);
1557                         M_MOV_IMM(1, REG_ITMP1);
1558                         M_MOV_IMM(-1, REG_ITMP2);
1559                         emit_ucomiss_reg_reg(cd, s1, s2);
1560                         M_CMOVB(REG_ITMP1, d);
1561                         M_CMOVA(REG_ITMP2, d);
1562                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1563                         emit_store_dst(jd, iptr, d);
1564                         break;
1565
1566                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1567                                           /* == => 0, < => 1, > => -1 */
1568
1569                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1570                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1571                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1572                         M_CLR(d);
1573                         M_MOV_IMM(1, REG_ITMP1);
1574                         M_MOV_IMM(-1, REG_ITMP2);
1575                         emit_ucomisd_reg_reg(cd, s1, s2);
1576                         M_CMOVB(REG_ITMP1, d);
1577                         M_CMOVA(REG_ITMP2, d);
1578                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1579                         emit_store_dst(jd, iptr, d);
1580                         break;
1581
1582                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1583                                           /* == => 0, < => 1, > => -1 */
1584
1585                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1586                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1587                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1588                         M_CLR(d);
1589                         M_MOV_IMM(1, REG_ITMP1);
1590                         M_MOV_IMM(-1, REG_ITMP2);
1591                         emit_ucomisd_reg_reg(cd, s1, s2);
1592                         M_CMOVB(REG_ITMP1, d);
1593                         M_CMOVA(REG_ITMP2, d);
1594                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1595                         emit_store_dst(jd, iptr, d);
1596                         break;
1597
1598
1599                 /* memory operations **************************************************/
1600
1601                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1602
1603                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1604                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1605                         gen_nullptr_check(s1);
1606                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1607                         emit_store_dst(jd, iptr, d);
1608                         break;
1609
1610                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1611
1612                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1613                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1614                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1615                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1616                                 gen_nullptr_check(s1);
1617                                 gen_bound_check;
1618                         }
1619                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1620                         emit_store_dst(jd, iptr, d);
1621                         break;
1622
1623                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1624
1625                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1626                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1627                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1628                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1629                                 gen_nullptr_check(s1);
1630                                 gen_bound_check;
1631                         }
1632                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1633                         emit_store_dst(jd, iptr, d);
1634                         break;
1635
1636                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1637
1638                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1639                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1640                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1641                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1642                                 gen_nullptr_check(s1);
1643                                 gen_bound_check;
1644                         }
1645                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1646                         emit_store_dst(jd, iptr, d);
1647                         break;
1648
1649                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1650
1651                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1652                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1653                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1654                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1655                                 gen_nullptr_check(s1);
1656                                 gen_bound_check;
1657                         }
1658                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1659                         emit_store_dst(jd, iptr, d);
1660                         break;
1661
1662                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1663
1664                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1665                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1666                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1667                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1668                                 gen_nullptr_check(s1);
1669                                 gen_bound_check;
1670                         }
1671                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1672                         emit_store_dst(jd, iptr, d);
1673                         break;
1674
1675                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1676
1677                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1678                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1679                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1680                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1681                                 gen_nullptr_check(s1);
1682                                 gen_bound_check;
1683                         }
1684                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1685                         emit_store_dst(jd, iptr, d);
1686                         break;
1687
1688                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1689
1690                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1691                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1692                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1693                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1694                                 gen_nullptr_check(s1);
1695                                 gen_bound_check;
1696                         }
1697                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1698                         emit_store_dst(jd, iptr, d);
1699                         break;
1700
1701                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1702
1703                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1704                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1705                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1706                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1707                                 gen_nullptr_check(s1);
1708                                 gen_bound_check;
1709                         }
1710                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1711                         emit_store_dst(jd, iptr, d);
1712                         break;
1713
1714
1715                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1716
1717                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1718                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1719                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1720                                 gen_nullptr_check(s1);
1721                                 gen_bound_check;
1722                         }
1723                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1724                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1725                         break;
1726
1727                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1728
1729                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1730                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1731                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1732                                 gen_nullptr_check(s1);
1733                                 gen_bound_check;
1734                         }
1735                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1736                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1737                         break;
1738
1739                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1740
1741                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1742                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1743                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1744                                 gen_nullptr_check(s1);
1745                                 gen_bound_check;
1746                         }
1747                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1748                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1749                         break;
1750
1751                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1752
1753                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1754                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1755                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1756                                 gen_nullptr_check(s1);
1757                                 gen_bound_check;
1758                         }
1759                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1760                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
1761                         break;
1762
1763                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1764
1765                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1766                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1767                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1768                                 gen_nullptr_check(s1);
1769                                 gen_bound_check;
1770                         }
1771                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1772                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
1773                         break;
1774
1775                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1776
1777                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1778                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1779                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1780                                 gen_nullptr_check(s1);
1781                                 gen_bound_check;
1782                         }
1783                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1784                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
1785                         break;
1786
1787                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1788
1789                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1790                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1791                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1792                                 gen_nullptr_check(s1);
1793                                 gen_bound_check;
1794                         }
1795                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1796                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
1797                         break;
1798
1799                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1800
1801                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1802                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1803                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1804                                 gen_nullptr_check(s1);
1805                                 gen_bound_check;
1806                         }
1807                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1808
1809                         M_MOV(s1, REG_A0);
1810                         M_MOV(s3, REG_A1);
1811                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
1812                         M_CALL(REG_ITMP1);
1813                         M_TEST(REG_RESULT);
1814                         M_BEQ(0);
1815                         codegen_add_arraystoreexception_ref(cd);
1816
1817                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1818                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1819                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1820                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1821                         break;
1822
1823
1824                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1825
1826                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1827                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1828                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1829                                 gen_nullptr_check(s1);
1830                                 gen_bound_check;
1831                         }
1832                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1833                         break;
1834
1835                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1836
1837                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1838                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1839                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1840                                 gen_nullptr_check(s1);
1841                                 gen_bound_check;
1842                         }
1843                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray, data[0]), s1, s2, 1);
1844                         break;
1845
1846                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1847
1848                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1849                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1850                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1851                                 gen_nullptr_check(s1);
1852                                 gen_bound_check;
1853                         }
1854                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1855                         break;
1856
1857                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1858
1859                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1860                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1861                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1862                                 gen_nullptr_check(s1);
1863                                 gen_bound_check;
1864                         }
1865                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray, data[0]), s1, s2, 2);
1866                         break;
1867
1868                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1869
1870                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1871                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1872                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1873                                 gen_nullptr_check(s1);
1874                                 gen_bound_check;
1875                         }
1876
1877                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1878                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1879                         } else {
1880                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1881                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
1882                         }
1883                         break;
1884
1885                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1886
1887                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1888                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1889                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1890                                 gen_nullptr_check(s1);
1891                                 gen_bound_check;
1892                         }
1893                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1894                         break;
1895
1896
1897                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1898
1899                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1900                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
1901
1902                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1903                                 disp = dseg_addaddress(cd, NULL);
1904                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1905
1906                                 /* must be calculated before codegen_addpatchref */
1907
1908                                 if (opt_showdisassemble)
1909                                         disp -= PATCHER_CALL_SIZE;
1910
1911 /*                              PROFILE_CYCLE_STOP; */
1912
1913                                 codegen_addpatchref(cd, PATCHER_get_putstatic, uf, disp);
1914
1915                                 if (opt_showdisassemble) {
1916                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
1917                                 }
1918
1919 /*                              PROFILE_CYCLE_START; */
1920                         }
1921                         else {
1922                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
1923
1924                                 fieldtype = fi->type;
1925
1926                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1927                                         PROFILE_CYCLE_STOP;
1928
1929                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
1930
1931                                         if (opt_showdisassemble) {
1932                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
1933                                         }
1934
1935                                         PROFILE_CYCLE_START;
1936                                 }
1937
1938                                 disp = dseg_addaddress(cd, &(fi->value));
1939                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1940                         }
1941
1942                         /* This approach is much faster than moving the field
1943                            address inline into a register. */
1944
1945                         M_ALD(REG_ITMP1, RIP, disp);
1946
1947                         switch (fieldtype) {
1948                         case TYPE_INT:
1949                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1950                                 M_ILD(d, REG_ITMP1, 0);
1951                                 break;
1952                         case TYPE_LNG:
1953                         case TYPE_ADR:
1954                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1955                                 M_LLD(d, REG_ITMP1, 0);
1956                                 break;
1957                         case TYPE_FLT:
1958                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1959                                 M_FLD(d, REG_ITMP1, 0);
1960                                 break;
1961                         case TYPE_DBL:
1962                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1963                                 M_DLD(d, REG_ITMP1, 0);
1964                                 break;
1965                         }
1966                         emit_store_dst(jd, iptr, d);
1967                         break;
1968
1969                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1970
1971                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1972                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
1973
1974                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1975                                 disp = dseg_addaddress(cd, NULL);
1976                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1977
1978                                 /* must be calculated before codegen_addpatchref */
1979
1980                                 if (opt_showdisassemble)
1981                                         disp -= PATCHER_CALL_SIZE;
1982
1983 /*                              PROFILE_CYCLE_STOP; */
1984
1985                                 codegen_addpatchref(cd, PATCHER_get_putstatic, uf, disp);
1986
1987                                 if (opt_showdisassemble) {
1988                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
1989                                 }
1990
1991 /*                              PROFILE_CYCLE_START; */
1992                         }
1993                         else {
1994                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
1995
1996                                 fieldtype = fi->type;
1997
1998                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1999                                         PROFILE_CYCLE_STOP;
2000
2001                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2002
2003                                         if (opt_showdisassemble) {
2004                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2005                                                 disp -= PATCHER_CALL_SIZE;
2006                                         }
2007
2008                                         PROFILE_CYCLE_START;
2009                                 }
2010
2011                                 disp = dseg_addaddress(cd, &(fi->value));
2012                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2013                         }
2014
2015                         /* This approach is much faster than moving the field
2016                            address inline into a register. */
2017
2018                         M_ALD(REG_ITMP1, RIP, disp);
2019
2020                         switch (fieldtype) {
2021                         case TYPE_INT:
2022                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2023                                 M_IST(s1, REG_ITMP1, 0);
2024                                 break;
2025                         case TYPE_LNG:
2026                         case TYPE_ADR:
2027                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2028                                 M_LST(s1, REG_ITMP1, 0);
2029                                 break;
2030                         case TYPE_FLT:
2031                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
2032                                 M_FST(s1, REG_ITMP1, 0);
2033                                 break;
2034                         case TYPE_DBL:
2035                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
2036                                 M_DST(s1, REG_ITMP1, 0);
2037                                 break;
2038                         }
2039                         break;
2040
2041                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2042                                           /* val = value (in current instruction)     */
2043                                           /* following NOP)                           */
2044
2045                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2046                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2047
2048                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2049                                 disp = dseg_addaddress(cd, NULL);
2050                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2051
2052                                 /* must be calculated before codegen_addpatchref */
2053
2054                                 if (opt_showdisassemble)
2055                                         disp -= PATCHER_CALL_SIZE;
2056
2057
2058 /*                              PROFILE_CYCLE_STOP; */
2059
2060                                 codegen_addpatchref(cd, PATCHER_get_putstatic, uf, disp);
2061
2062                                 if (opt_showdisassemble) {
2063                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2064                                 }
2065
2066 /*                              PROFILE_CYCLE_START; */
2067                         }
2068                         else {
2069                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2070
2071                                 fieldtype = fi->type;
2072
2073                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2074                                         PROFILE_CYCLE_STOP;
2075
2076                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2077
2078                                         if (opt_showdisassemble) {
2079                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2080                                         }
2081
2082                                         PROFILE_CYCLE_START;
2083                                 }
2084
2085                                 disp = dseg_addaddress(cd, &(fi->value));
2086                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2087                         }
2088
2089                         /* This approach is much faster than moving the field
2090                            address inline into a register. */
2091
2092                         M_ALD(REG_ITMP1, RIP, disp);
2093
2094                         switch (fieldtype) {
2095                         case TYPE_INT:
2096                         case TYPE_FLT:
2097                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2098                                 break;
2099                         case TYPE_LNG:
2100                         case TYPE_ADR:
2101                         case TYPE_DBL:
2102                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
2103                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2104                                 else {
2105                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2106                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
2107                                 }
2108                                 break;
2109                         }
2110                         break;
2111
2112                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2113
2114                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2115                         gen_nullptr_check(s1);
2116
2117                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2118                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2119
2120 /*                              PROFILE_CYCLE_STOP; */
2121
2122                                 codegen_addpatchref(cd, PATCHER_get_putfield, uf, 0);
2123
2124                                 if (opt_showdisassemble) {
2125                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2126                                 }
2127
2128 /*                              PROFILE_CYCLE_START; */
2129
2130                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2131                                 disp = 0;
2132                         }
2133                         else {
2134                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2135
2136                                 fieldtype = fi->type;
2137                                 disp = fi->offset;
2138                         }
2139
2140                         switch (fieldtype) {
2141                         case TYPE_INT:
2142                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
2143                                 M_ILD32(d, s1, disp);
2144                                 break;
2145                         case TYPE_LNG:
2146                         case TYPE_ADR:
2147                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
2148                                 M_LLD32(d, s1, disp);
2149                                 break;
2150                         case TYPE_FLT:
2151                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2152                                 M_FLD32(d, s1, disp);
2153                                 break;
2154                         case TYPE_DBL:
2155                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2156                                 M_DLD32(d, s1, disp);
2157                                 break;
2158                         }
2159                         emit_store_dst(jd, iptr, d);
2160                         break;
2161
2162                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2163
2164                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2165                         gen_nullptr_check(s1);
2166
2167                         s2 = emit_load_s2(jd, iptr, REG_IFTMP);
2168
2169                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2170                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2171
2172 /*                              PROFILE_CYCLE_STOP; */
2173
2174                                 codegen_addpatchref(cd, PATCHER_get_putfield, uf, 0);
2175
2176                                 if (opt_showdisassemble) {
2177                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2178                                 }
2179
2180 /*                              PROFILE_CYCLE_START; */
2181
2182                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2183                                 disp = 0;
2184                         }
2185                         else {
2186                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2187
2188                                 fieldtype = fi->type;
2189                                 disp = fi->offset;
2190                         }
2191
2192                         switch (fieldtype) {
2193                         case TYPE_INT:
2194                                 M_IST32(s2, s1, disp);
2195                                 break;
2196                         case TYPE_LNG:
2197                         case TYPE_ADR:
2198                                 M_LST32(s2, s1, disp);
2199                                 break;
2200                         case TYPE_FLT:
2201                                 M_FST32(s2, s1, disp);
2202                                 break;
2203                         case TYPE_DBL:
2204                                 M_DST32(s2, s1, disp);
2205                                 break;
2206                         }
2207                         break;
2208
2209                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2210                                           /* val = value (in current instruction)     */
2211                                           /* following NOP)                           */
2212
2213                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2214                         gen_nullptr_check(s1);
2215
2216                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2217                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2218
2219 /*                              PROFILE_CYCLE_STOP; */
2220
2221                                 codegen_addpatchref(cd, PATCHER_putfieldconst, uf, 0);
2222
2223                                 if (opt_showdisassemble) {
2224                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2225                                 }
2226
2227 /*                              PROFILE_CYCLE_START; */
2228
2229                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2230                                 disp = 0;
2231                         }
2232                         else {
2233                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2234
2235                                 fieldtype = fi->type;
2236                                 disp = fi->offset;
2237                         }
2238
2239                         switch (fieldtype) {
2240                         case TYPE_INT:
2241                         case TYPE_FLT:
2242                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2243                                 break;
2244                         case TYPE_LNG:
2245                         case TYPE_ADR:
2246                         case TYPE_DBL:
2247                                 /* XXX why no check for IS_IMM32? */
2248                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2249                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2250                                 break;
2251                         }
2252                         break;
2253
2254
2255                 /* branch operations **************************************************/
2256
2257                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2258
2259                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2260                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2261
2262                         PROFILE_CYCLE_STOP;
2263
2264 #ifdef ENABLE_VERIFIER
2265                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2266                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2267                                                                         iptr->sx.s23.s2.uc, 0);
2268
2269                                 if (opt_showdisassemble) {
2270                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2271                                 }
2272                         }
2273 #endif /* ENABLE_VERIFIER */
2274
2275                         M_CALL_IMM(0);                            /* passing exception pc */
2276                         M_POP(REG_ITMP2_XPC);
2277
2278                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2279                         M_JMP(REG_ITMP3);
2280                         break;
2281
2282                 case ICMD_GOTO:         /* ... ==> ...                                */
2283                 case ICMD_RET:          /* ... ==> ...                                */
2284
2285                         M_JMP_IMM(0);
2286                         codegen_addreference(cd, iptr->dst.block);
2287                         break;
2288
2289                 case ICMD_JSR:          /* ... ==> ...                                */
2290
2291                         M_JMP_IMM(0);
2292                         codegen_addreference(cd, iptr->sx.s23.s3.jsrtarget.block);
2293                         break;
2294
2295                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2296
2297                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2298                         M_TEST(s1);
2299                         M_BEQ(0);
2300                         codegen_addreference(cd, iptr->dst.block);
2301                         break;
2302
2303                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2304
2305                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2306                         M_TEST(s1);
2307                         M_BNE(0);
2308                         codegen_addreference(cd, iptr->dst.block);
2309                         break;
2310
2311                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2312
2313                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2314                         M_ICMP_IMM(iptr->sx.val.i, s1);
2315                         M_BEQ(0);
2316                         codegen_addreference(cd, iptr->dst.block);
2317                         break;
2318
2319                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2320
2321                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2322                         M_ICMP_IMM(iptr->sx.val.i, s1);
2323                         M_BLT(0);
2324                         codegen_addreference(cd, iptr->dst.block);
2325                         break;
2326
2327                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2328
2329                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2330                         M_ICMP_IMM(iptr->sx.val.i, s1);
2331                         M_BLE(0);
2332                         codegen_addreference(cd, iptr->dst.block);
2333                         break;
2334
2335                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2336
2337                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2338                         M_ICMP_IMM(iptr->sx.val.i, s1);
2339                         M_BNE(0);
2340                         codegen_addreference(cd, iptr->dst.block);
2341                         break;
2342
2343                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2344
2345                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2346                         M_ICMP_IMM(iptr->sx.val.i, s1);
2347                         M_BGT(0);
2348                         codegen_addreference(cd, iptr->dst.block);
2349                         break;
2350
2351                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2352
2353                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2354                         M_ICMP_IMM(iptr->sx.val.i, s1);
2355                         M_BGE(0);
2356                         codegen_addreference(cd, iptr->dst.block);
2357                         break;
2358
2359                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2360
2361                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2362                         if (IS_IMM32(iptr->sx.val.l))
2363                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2364                         else {
2365                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2366                                 M_LCMP(REG_ITMP2, s1);
2367                         }
2368                         M_BEQ(0);
2369                         codegen_addreference(cd, iptr->dst.block);
2370                         break;
2371
2372                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2373
2374                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2375                         if (IS_IMM32(iptr->sx.val.l))
2376                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2377                         else {
2378                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2379                                 M_LCMP(REG_ITMP2, s1);
2380                         }
2381                         M_BLT(0);
2382                         codegen_addreference(cd, iptr->dst.block);
2383                         break;
2384
2385                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2386
2387                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2388                         if (IS_IMM32(iptr->sx.val.l))
2389                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2390                         else {
2391                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2392                                 M_LCMP(REG_ITMP2, s1);
2393                         }
2394                         M_BLE(0);
2395                         codegen_addreference(cd, iptr->dst.block);
2396                         break;
2397
2398                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2399
2400                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2401                         if (IS_IMM32(iptr->sx.val.l))
2402                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2403                         else {
2404                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2405                                 M_LCMP(REG_ITMP2, s1);
2406                         }
2407                         M_BNE(0);
2408                         codegen_addreference(cd, iptr->dst.block);
2409                         break;
2410
2411                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2412
2413                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2414                         if (IS_IMM32(iptr->sx.val.l))
2415                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2416                         else {
2417                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2418                                 M_LCMP(REG_ITMP2, s1);
2419                         }
2420                         M_BGT(0);
2421                         codegen_addreference(cd, iptr->dst.block);
2422                         break;
2423
2424                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2425
2426                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2427                         if (IS_IMM32(iptr->sx.val.l))
2428                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2429                         else {
2430                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2431                                 M_LCMP(REG_ITMP2, s1);
2432                         }
2433                         M_BGE(0);
2434                         codegen_addreference(cd, iptr->dst.block);
2435                         break;
2436
2437                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2438
2439                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2440                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2441                         M_ICMP(s2, s1);
2442                         M_BEQ(0);
2443                         codegen_addreference(cd, iptr->dst.block);
2444                         break;
2445
2446                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2447                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2448
2449                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2450                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2451                         M_LCMP(s2, s1);
2452                         M_BEQ(0);
2453                         codegen_addreference(cd, iptr->dst.block);
2454                         break;
2455
2456                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2457
2458                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2459                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2460                         M_ICMP(s2, s1);
2461                         M_BNE(0);
2462                         codegen_addreference(cd, iptr->dst.block);
2463                         break;
2464
2465                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2466                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2467
2468                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2469                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2470                         M_LCMP(s2, s1);
2471                         M_BNE(0);
2472                         codegen_addreference(cd, iptr->dst.block);
2473                         break;
2474
2475                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2476
2477                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2478                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2479                         M_ICMP(s2, s1);
2480                         M_BLT(0);
2481                         codegen_addreference(cd, iptr->dst.block);
2482                         break;
2483
2484                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2485
2486                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2487                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2488                         M_LCMP(s2, s1);
2489                         M_BLT(0);
2490                         codegen_addreference(cd, iptr->dst.block);
2491                         break;
2492
2493                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2494
2495                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2496                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2497                         M_ICMP(s2, s1);
2498                         M_BGT(0);
2499                         codegen_addreference(cd, iptr->dst.block);
2500                         break;
2501
2502                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2503
2504                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2505                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2506                         M_LCMP(s2, s1);
2507                         M_BGT(0);
2508                         codegen_addreference(cd, iptr->dst.block);
2509                         break;
2510
2511                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2512
2513                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2514                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2515                         M_ICMP(s2, s1);
2516                         M_BLE(0);
2517                         codegen_addreference(cd, iptr->dst.block);
2518                         break;
2519
2520                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2521
2522                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2523                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2524                         M_LCMP(s2, s1);
2525                         M_BLE(0);
2526                         codegen_addreference(cd, iptr->dst.block);
2527                         break;
2528
2529                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2530
2531                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2532                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2533                         M_ICMP(s2, s1);
2534                         M_BGE(0);
2535                         codegen_addreference(cd, iptr->dst.block);
2536                         break;
2537
2538                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2539
2540                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2541                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2542                         M_LCMP(s2, s1);
2543                         M_BGE(0);
2544                         codegen_addreference(cd, iptr->dst.block);
2545                         break;
2546
2547                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2548                 case ICMD_LRETURN:
2549
2550                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2551                         M_INTMOVE(s1, REG_RESULT);
2552                         goto nowperformreturn;
2553
2554                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2555
2556                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2557                         M_INTMOVE(s1, REG_RESULT);
2558
2559 #ifdef ENABLE_VERIFIER
2560                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2561                                 PROFILE_CYCLE_STOP;
2562
2563                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2564                                                                         iptr->sx.s23.s2.uc, 0);
2565
2566                                 if (opt_showdisassemble) {
2567                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2568                                 }
2569
2570                                 PROFILE_CYCLE_START;
2571                         }
2572 #endif /* ENABLE_VERIFIER */
2573                         goto nowperformreturn;
2574
2575                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2576                 case ICMD_DRETURN:
2577
2578                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2579                         M_FLTMOVE(s1, REG_FRESULT);
2580                         goto nowperformreturn;
2581
2582                 case ICMD_RETURN:      /* ...  ==> ...                                */
2583
2584 nowperformreturn:
2585                         {
2586                         s4 i, p;
2587
2588                         p = cd->stackframesize;
2589
2590 #if !defined(NDEBUG)
2591                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2592                                 emit_verbosecall_exit(jd);
2593 #endif /* !defined(NDEBUG) */
2594
2595 #if defined(ENABLE_THREADS)
2596                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2597                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2598
2599                                 /* we need to save the proper return value */
2600                                 switch (iptr->opc) {
2601                                 case ICMD_IRETURN:
2602                                 case ICMD_ARETURN:
2603                                 case ICMD_LRETURN:
2604                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2605                                         break;
2606                                 case ICMD_FRETURN:
2607                                 case ICMD_DRETURN:
2608                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2609                                         break;
2610                                 }
2611
2612                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2613                                 M_CALL(REG_ITMP1);
2614
2615                                 /* and now restore the proper return value */
2616                                 switch (iptr->opc) {
2617                                 case ICMD_IRETURN:
2618                                 case ICMD_ARETURN:
2619                                 case ICMD_LRETURN:
2620                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2621                                         break;
2622                                 case ICMD_FRETURN:
2623                                 case ICMD_DRETURN:
2624                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2625                                         break;
2626                                 }
2627                         }
2628 #endif
2629
2630                         /* restore saved registers */
2631
2632                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2633                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2634                         }
2635                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2636                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2637                         }
2638
2639                         /* deallocate stack */
2640
2641                         if (cd->stackframesize)
2642                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2643
2644                         /* generate method profiling code */
2645
2646                         PROFILE_CYCLE_STOP;
2647
2648                         M_RET;
2649                         }
2650                         break;
2651
2652
2653                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2654                         {
2655                                 s4 i, l;
2656                                 branch_target_t *table;
2657
2658                                 table = iptr->dst.table;
2659
2660                                 l = iptr->sx.s23.s2.tablelow;
2661                                 i = iptr->sx.s23.s3.tablehigh;
2662
2663                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2664                                 M_INTMOVE(s1, REG_ITMP1);
2665
2666                                 if (l != 0)
2667                                         M_ISUB_IMM(l, REG_ITMP1);
2668
2669                                 /* number of targets */
2670                                 i = i - l + 1;
2671
2672                 /* range check */
2673                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2674                                 M_BA(0);
2675
2676                                 codegen_addreference(cd, table[0].block); /* default target */
2677
2678                                 /* build jump table top down and use address of lowest entry */
2679
2680                                 table += i;
2681
2682                                 while (--i >= 0) {
2683                                         dseg_addtarget(cd, table->block);
2684                                         --table;
2685                                 }
2686
2687                                 /* length of dataseg after last dseg_addtarget is used
2688                                    by load */
2689
2690                                 M_MOV_IMM(0, REG_ITMP2);
2691                                 dseg_adddata(cd);
2692                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2693                                 M_JMP(REG_ITMP1);
2694                         }
2695                         break;
2696
2697
2698                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2699                         {
2700                                 s4 i;
2701                                 lookup_target_t *lookup;
2702
2703                                 lookup = iptr->dst.lookup;
2704
2705                                 i = iptr->sx.s23.s2.lookupcount;
2706
2707                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2708                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2709
2710                                 while (--i >= 0) {
2711                                         M_ICMP_IMM(lookup->value, s1);
2712                                         M_BEQ(0);
2713                                         codegen_addreference(cd, lookup->target.block);
2714                                         lookup++;
2715                                 }
2716
2717                                 M_JMP_IMM(0);
2718
2719                                 codegen_addreference(cd, iptr->sx.s23.s3.lookupdefault.block);
2720                         }
2721                         break;
2722
2723
2724                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2725
2726                         bte = iptr->sx.s23.s3.bte;
2727                         md  = bte->md;
2728                         goto gen_method;
2729
2730                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2731
2732                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2733                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2734                 case ICMD_INVOKEINTERFACE:
2735
2736                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2737                                 lm = NULL;
2738                                 um = iptr->sx.s23.s3.um;
2739                                 md = um->methodref->parseddesc.md;
2740                         }
2741                         else {
2742                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2743                                 um = NULL;
2744                                 md = lm->parseddesc;
2745                         }
2746
2747 gen_method:
2748                         s3 = md->paramcount;
2749
2750                         MCODECHECK((20 * s3) + 128);
2751
2752                         /* copy arguments to registers or stack location */
2753
2754                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2755                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2756
2757                                 /* Already Preallocated (ARGVAR) ? */
2758                                 if (var->flags & PREALLOC)
2759                                         continue;
2760
2761                                 if (IS_INT_LNG_TYPE(var->type)) {
2762                                         if (!md->params[s3].inmemory) {
2763                                                 s1 = rd->argintregs[md->params[s3].regoff];
2764                                                 d = emit_load(jd, iptr, var, s1);
2765                                                 M_INTMOVE(d, s1);
2766                                         }
2767                                         else {
2768                                                 d = emit_load(jd, iptr, var, REG_ITMP1);
2769                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
2770                                         }
2771                                 }
2772                                 else {
2773                                         if (!md->params[s3].inmemory) {
2774                                                 s1 = rd->argfltregs[md->params[s3].regoff];
2775                                                 d = emit_load(jd, iptr, var, s1);
2776                                                 M_FLTMOVE(d, s1);
2777                                         }
2778                                         else {
2779                                                 d = emit_load(jd, iptr, var, REG_FTMP1);
2780
2781                                                 if (IS_2_WORD_TYPE(var->type))
2782                                                         M_DST(d, REG_SP, md->params[s3].regoff * 8);
2783                                                 else
2784                                                         M_FST(d, REG_SP, md->params[s3].regoff * 8);
2785                                         }
2786                                 }
2787                         }
2788
2789                         /* generate method profiling code */
2790
2791                         PROFILE_CYCLE_STOP;
2792
2793                         switch (iptr->opc) {
2794                         case ICMD_BUILTIN:
2795                                 M_MOV_IMM(bte->fp, REG_ITMP1);
2796                                 M_CALL(REG_ITMP1);
2797
2798                                 if (INSTRUCTION_MUST_CHECK(iptr)) {
2799                                         M_TEST(REG_RESULT);
2800                                         M_BEQ(0);
2801                                         codegen_add_fillinstacktrace_ref(cd);
2802                                 }
2803                                 break;
2804
2805                         case ICMD_INVOKESPECIAL:
2806                                 M_TEST(REG_A0);
2807                                 M_BEQ(0);
2808                                 codegen_add_nullpointerexception_ref(cd);
2809
2810                                 /* fall through */
2811
2812                         case ICMD_INVOKESTATIC:
2813                                 if (lm == NULL) {
2814                                         disp = dseg_addaddress(cd, NULL);
2815                                         disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2816
2817                                         /* must be calculated before codegen_addpatchref */
2818
2819                                         if (opt_showdisassemble)
2820                                                 disp -= PATCHER_CALL_SIZE;
2821
2822                                         codegen_addpatchref(cd, PATCHER_invokestatic_special,
2823                                                                                 um, disp);
2824
2825                                         if (opt_showdisassemble) {
2826                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2827                                         }
2828
2829 /*                                      a = 0; */
2830                                 }
2831                                 else {
2832                                         disp = dseg_addaddress(cd, lm->stubroutine);
2833                                         disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2834
2835 /*                                      a = (ptrint) lm->stubroutine; */
2836                                 }
2837
2838 /*                              M_MOV_IMM(a, REG_ITMP2); */
2839                                 M_ALD(REG_ITMP2, RIP, disp);
2840                                 M_CALL(REG_ITMP2);
2841                                 break;
2842
2843                         case ICMD_INVOKEVIRTUAL:
2844                                 gen_nullptr_check(REG_A0);
2845
2846                                 if (lm == NULL) {
2847                                         codegen_addpatchref(cd, PATCHER_invokevirtual, um, 0);
2848
2849                                         if (opt_showdisassemble) {
2850                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2851                                         }
2852
2853                                         s1 = 0;
2854                                 }
2855                                 else
2856                                         s1 = OFFSET(vftbl_t, table[0]) +
2857                                                 sizeof(methodptr) * lm->vftblindex;
2858
2859                                 M_ALD(REG_METHODPTR, REG_A0,
2860                                           OFFSET(java_objectheader, vftbl));
2861                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2862                                 M_CALL(REG_ITMP3);
2863                                 break;
2864
2865                         case ICMD_INVOKEINTERFACE:
2866                                 gen_nullptr_check(REG_A0);
2867
2868                                 if (lm == NULL) {
2869                                         codegen_addpatchref(cd, PATCHER_invokeinterface, um, 0);
2870
2871                                         if (opt_showdisassemble) {
2872                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2873                                         }
2874
2875                                         s1 = 0;
2876                                         s2 = 0;
2877                                 }
2878                                 else {
2879                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2880                                                 sizeof(methodptr) * lm->class->index;
2881
2882                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2883                                 }
2884
2885                                 M_ALD(REG_METHODPTR, REG_A0,
2886                                           OFFSET(java_objectheader, vftbl));
2887                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2888                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2889                                 M_CALL(REG_ITMP3);
2890                                 break;
2891                         }
2892
2893                         /* generate method profiling code */
2894
2895                         PROFILE_CYCLE_START;
2896
2897                         /* store return value */
2898
2899                         d = md->returntype.type;
2900
2901                         if (d != TYPE_VOID) {
2902                                 if (IS_INT_LNG_TYPE(d)) {
2903                                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2904                                         M_INTMOVE(REG_RESULT, s1);
2905                                 }
2906                                 else {
2907                                         s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2908                                         M_FLTMOVE(REG_FRESULT, s1);
2909                                 }
2910                                 emit_store_dst(jd, iptr, s1);
2911                         }
2912                         break;
2913
2914
2915                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2916
2917                                       /* val.a: (classinfo *) superclass              */
2918
2919                         /*  superclass is an interface:
2920                          *
2921                          *  OK if ((sub == NULL) ||
2922                          *         (sub->vftbl->interfacetablelength > super->index) &&
2923                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
2924                          *
2925                          *  superclass is a class:
2926                          *
2927                          *  OK if ((sub == NULL) || (0
2928                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
2929                          *         super->vftbl->diffval));
2930                          */
2931
2932                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2933                                 /* object type cast-check */
2934
2935                                 classinfo *super;
2936                                 vftbl_t   *supervftbl;
2937                                 s4         superindex;
2938
2939                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2940                                         super = NULL;
2941                                         superindex = 0;
2942                                         supervftbl = NULL;
2943                                 }
2944                                 else {
2945                                         super = iptr->sx.s23.s3.c.cls;
2946                                         superindex = super->index;
2947                                         supervftbl = super->vftbl;
2948                                 }
2949
2950 #if defined(ENABLE_THREADS)
2951                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
2952 #endif
2953                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2954
2955                                 /* calculate interface checkcast code size */
2956
2957                                 s2 = 3; /* mov_membase_reg */
2958                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
2959
2960                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
2961                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
2962                                         3 /* test */ + 6 /* jcc */;
2963
2964                                 if (super == NULL)
2965                                         s2 += (opt_showdisassemble ? 5 : 0);
2966
2967                                 /* calculate class checkcast code size */
2968
2969                                 s3 = 3; /* mov_membase_reg */
2970                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
2971                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
2972
2973 #if 0
2974                                 if (s1 != REG_ITMP1) {
2975                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
2976                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
2977                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
2978                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
2979                                         a += 3;    /* sub */
2980
2981                                 } else
2982 #endif
2983                                         {
2984                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
2985                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
2986                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2987                                         }
2988
2989                                 s3 += 3 /* cmp */ + 6 /* jcc */;
2990
2991                                 if (super == NULL)
2992                                         s3 += (opt_showdisassemble ? 5 : 0);
2993
2994                                 /* if class is not resolved, check which code to call */
2995
2996                                 if (super == NULL) {
2997                                         M_TEST(s1);
2998                                         M_BEQ(6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
2999
3000                                         codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3001                                                                                 iptr->sx.s23.s3.c.ref, 0);
3002
3003                                         if (opt_showdisassemble) {
3004                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3005                                         }
3006
3007                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
3008                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
3009                                         M_BEQ(s2 + 5);
3010                                 }
3011
3012                                 /* interface checkcast code */
3013
3014                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
3015                                         if (super != NULL) {
3016                                                 M_TEST(s1);
3017                                                 M_BEQ(s2);
3018                                         }
3019
3020                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3021
3022                                         if (super == NULL) {
3023                                                 codegen_addpatchref(cd,
3024                                                                                         PATCHER_checkcast_instanceof_interface,
3025                                                                                         iptr->sx.s23.s3.c.ref,
3026                                                                                         0);
3027
3028                                                 if (opt_showdisassemble) {
3029                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3030                                                 }
3031                                         }
3032
3033                                         emit_movl_membase32_reg(cd, REG_ITMP2,
3034                                                                                           OFFSET(vftbl_t, interfacetablelength),
3035                                                                                           REG_ITMP3);
3036                                         /* XXX TWISTI: should this be int arithmetic? */
3037                                         M_LSUB_IMM32(superindex, REG_ITMP3);
3038                                         M_TEST(REG_ITMP3);
3039                                         M_BLE(0);
3040                                         codegen_add_classcastexception_ref(cd, s1);
3041                                         emit_mov_membase32_reg(cd, REG_ITMP2,
3042                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
3043                                                                                          superindex * sizeof(methodptr*),
3044                                                                                          REG_ITMP3);
3045                                         M_TEST(REG_ITMP3);
3046                                         M_BEQ(0);
3047                                         codegen_add_classcastexception_ref(cd, s1);
3048
3049                                         if (super == NULL)
3050                                                 M_JMP_IMM(s3);
3051                                 }
3052
3053                                 /* class checkcast code */
3054
3055                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
3056                                         if (super != NULL) {
3057                                                 M_TEST(s1);
3058                                                 M_BEQ(s3);
3059                                         }
3060
3061                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3062
3063                                         if (super == NULL) {
3064                                                 codegen_addpatchref(cd, PATCHER_checkcast_class,
3065                                                                                         iptr->sx.s23.s3.c.ref,
3066                                                                                         0);
3067
3068                                                 if (opt_showdisassemble) {
3069                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3070                                                 }
3071                                         }
3072
3073                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3074 #if defined(ENABLE_THREADS)
3075                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3076 #endif
3077                                         emit_movl_membase32_reg(cd, REG_ITMP2,
3078                                                                                           OFFSET(vftbl_t, baseval),
3079                                                                                           REG_ITMP2);
3080                                         /*                                      if (s1 != REG_ITMP1) { */
3081                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3082                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
3083                                         /*                                                                                              REG_ITMP1); */
3084                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3085                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3086                                         /*                                                                                              REG_ITMP3); */
3087                                         /*  #if defined(ENABLE_THREADS) */
3088                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3089                                         /*  #endif */
3090                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
3091
3092                                         /*                                      } else { */
3093                                         emit_movl_membase32_reg(cd, REG_ITMP3,
3094                                                                                           OFFSET(vftbl_t, baseval),
3095                                                                                           REG_ITMP3);
3096                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3097                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3098                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3099                                         /*                                      } */
3100 #if defined(ENABLE_THREADS)
3101                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3102 #endif
3103                                         M_LCMP(REG_ITMP3, REG_ITMP2);
3104                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3105                                         codegen_add_classcastexception_ref(cd, s1);
3106                                 }
3107
3108                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
3109                         }
3110                         else {
3111                                 /* array type cast-check */
3112
3113                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
3114                                 M_INTMOVE(s1, REG_A0);
3115
3116                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3117                                         codegen_addpatchref(cd, PATCHER_builtin_arraycheckcast,
3118                                                                                 iptr->sx.s23.s3.c.ref, 0);
3119
3120                                         if (opt_showdisassemble) {
3121                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3122                                         }
3123                                 }
3124
3125                                 M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
3126                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
3127                                 M_CALL(REG_ITMP1);
3128
3129                                 /* s1 may have been destroyed over the function call */
3130                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
3131                                 M_TEST(REG_RESULT);
3132                                 M_BEQ(0);
3133                                 codegen_add_classcastexception_ref(cd, s1);
3134
3135                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
3136                         }
3137
3138                         M_INTMOVE(s1, d);
3139                         emit_store_dst(jd, iptr, d);
3140                         break;
3141
3142                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3143
3144                                       /* val.a: (classinfo *) superclass              */
3145
3146                         /*  superclass is an interface:
3147                          *
3148                          *  return (sub != NULL) &&
3149                          *         (sub->vftbl->interfacetablelength > super->index) &&
3150                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3151                          *
3152                          *  superclass is a class:
3153                          *
3154                          *  return ((sub != NULL) && (0
3155                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3156                          *          super->vftbl->diffvall));
3157                          */
3158
3159                         {
3160                         classinfo *super;
3161                         vftbl_t   *supervftbl;
3162                         s4         superindex;
3163
3164                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3165                                 super = NULL;
3166                                 superindex = 0;
3167                                 supervftbl = NULL;
3168
3169                         } else {
3170                                 super = iptr->sx.s23.s3.c.cls;
3171                                 superindex = super->index;
3172                                 supervftbl = super->vftbl;
3173                         }
3174
3175 #if defined(ENABLE_THREADS)
3176             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3177 #endif
3178
3179                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
3180                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
3181                         if (s1 == d) {
3182                                 M_INTMOVE(s1, REG_ITMP1);
3183                                 s1 = REG_ITMP1;
3184                         }
3185
3186                         /* calculate interface instanceof code size */
3187
3188                         s2 = 3; /* mov_membase_reg */
3189                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3190                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3191                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3192                                 3 /* test */ + 4 /* setcc */;
3193
3194                         if (!super)
3195                                 s2 += (opt_showdisassemble ? 5 : 0);
3196
3197                         /* calculate class instanceof code size */
3198
3199                         s3 = 3; /* mov_membase_reg */
3200                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3201                         s3 += 10; /* mov_imm_reg */
3202                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3203                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3204                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3205                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3206                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3207                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3208                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3209
3210                         if (!super)
3211                                 s3 += (opt_showdisassemble ? 5 : 0);
3212
3213                         emit_alu_reg_reg(cd, ALU_XOR, d, d);
3214
3215                         /* if class is not resolved, check which code to call */
3216
3217                         if (!super) {
3218                                 emit_test_reg_reg(cd, s1, s1);
3219                                 emit_jcc(cd, CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3220                                                                                          7 + 6 + s2 + 5 + s3));
3221
3222                                 codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3223                                                                         iptr->sx.s23.s3.c.ref, 0);
3224
3225                                 if (opt_showdisassemble) {
3226                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3227                                 }
3228
3229                                 emit_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3230                                 emit_alul_imm_reg(cd, ALU_AND, ACC_INTERFACE, REG_ITMP3);
3231                                 emit_jcc(cd, CC_Z, s2 + 5);
3232                         }
3233
3234                         /* interface instanceof code */
3235
3236                         if (!super || (super->flags & ACC_INTERFACE)) {
3237                                 if (super) {
3238                                         emit_test_reg_reg(cd, s1, s1);
3239                                         emit_jcc(cd, CC_Z, s2);
3240                                 }
3241
3242                                 emit_mov_membase_reg(cd, s1,
3243                                                                            OFFSET(java_objectheader, vftbl),
3244                                                                            REG_ITMP1);
3245                                 if (!super) {
3246                                         codegen_addpatchref(cd,
3247                                                                                 PATCHER_checkcast_instanceof_interface,
3248                                                                                 iptr->sx.s23.s3.c.ref, 0);
3249
3250                                         if (opt_showdisassemble) {
3251                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3252                                         }
3253                                 }
3254
3255                                 emit_movl_membase32_reg(cd, REG_ITMP1,
3256                                                                                   OFFSET(vftbl_t, interfacetablelength),
3257                                                                                   REG_ITMP3);
3258                                 emit_alu_imm32_reg(cd, ALU_SUB, superindex, REG_ITMP3);
3259                                 emit_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3260
3261                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3262
3263                                 emit_jcc(cd, CC_LE, a);
3264                                 emit_mov_membase32_reg(cd, REG_ITMP1,
3265                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3266                                                                                  superindex * sizeof(methodptr*),
3267                                                                                  REG_ITMP1);
3268                                 emit_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3269                                 emit_setcc_reg(cd, CC_NE, d);
3270
3271                                 if (!super)
3272                                         emit_jmp_imm(cd, s3);
3273                         }
3274
3275                         /* class instanceof code */
3276
3277                         if (!super || !(super->flags & ACC_INTERFACE)) {
3278                                 if (super) {
3279                                         emit_test_reg_reg(cd, s1, s1);
3280                                         emit_jcc(cd, CC_E, s3);
3281                                 }
3282
3283                                 emit_mov_membase_reg(cd, s1,
3284                                                                            OFFSET(java_objectheader, vftbl),
3285                                                                            REG_ITMP1);
3286
3287                                 if (!super) {
3288                                         codegen_addpatchref(cd, PATCHER_instanceof_class,
3289                                                                                 iptr->sx.s23.s3.c.ref, 0);
3290
3291                                         if (opt_showdisassemble) {
3292                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3293                                         }
3294                                 }
3295
3296                                 emit_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3297 #if defined(ENABLE_THREADS)
3298                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3299 #endif
3300                                 emit_movl_membase_reg(cd, REG_ITMP1,
3301                                                                                 OFFSET(vftbl_t, baseval),
3302                                                                                 REG_ITMP1);
3303                                 emit_movl_membase_reg(cd, REG_ITMP2,
3304                                                                                 OFFSET(vftbl_t, diffval),
3305                                                                                 REG_ITMP3);
3306                                 emit_movl_membase_reg(cd, REG_ITMP2,
3307                                                                                 OFFSET(vftbl_t, baseval),
3308                                                                                 REG_ITMP2);
3309 #if defined(ENABLE_THREADS)
3310                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3311 #endif
3312                                 emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
3313                                 emit_alu_reg_reg(cd, ALU_XOR, d, d); /* may be REG_ITMP2 */
3314                                 emit_alu_reg_reg(cd, ALU_CMP, REG_ITMP3, REG_ITMP1);
3315                                 emit_setcc_reg(cd, CC_BE, d);
3316                         }
3317                         emit_store_dst(jd, iptr, d);
3318                         }
3319                         break;
3320
3321                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3322
3323                         /* check for negative sizes and copy sizes to stack if necessary  */
3324
3325                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
3326
3327                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
3328
3329                                 /* copy SAVEDVAR sizes to stack */
3330                                 var = VAR(iptr->sx.s23.s2.args[s1]);
3331
3332                                 /* Already Preallocated? */
3333                                 if (!(var->flags & PREALLOC)) {
3334                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
3335                                         M_LST(s2, REG_SP, s1 * 8);
3336                                 }
3337                         }
3338
3339                         /* is a patcher function set? */
3340
3341                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3342                                 codegen_addpatchref(cd, PATCHER_builtin_multianewarray,
3343                                                                         iptr->sx.s23.s3.c.ref, 0);
3344
3345                                 if (opt_showdisassemble) {
3346                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3347                                 }
3348                         }
3349
3350                         /* a0 = dimension count */
3351
3352                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
3353
3354                         /* a1 = classinfo */
3355
3356                         M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
3357
3358                         /* a2 = pointer to dimensions = stack pointer */
3359
3360                         M_MOV(REG_SP, REG_A2);
3361
3362                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3363                         M_CALL(REG_ITMP1);
3364
3365                         /* check for exception before result assignment */
3366
3367                         M_TEST(REG_RESULT);
3368                         M_BEQ(0);
3369                         codegen_add_fillinstacktrace_ref(cd);
3370
3371                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
3372                         M_INTMOVE(REG_RESULT, s1);
3373                         emit_store_dst(jd, iptr, s1);
3374                         break;
3375
3376                 default:
3377                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3378                         return false;
3379         } /* switch */
3380
3381         } /* for instruction */
3382
3383         MCODECHECK(512); /* XXX require a lower number? */
3384
3385         /* At the end of a basic block we may have to append some nops,
3386            because the patcher stub calling code might be longer than the
3387            actual instruction. So codepatching does not change the
3388            following block unintentionally. */
3389
3390         if (cd->mcodeptr < cd->lastmcodeptr) {
3391                 while (cd->mcodeptr < cd->lastmcodeptr) {
3392                         M_NOP;
3393                 }
3394         }
3395
3396         } /* if (bptr -> flags >= BBREACHED) */
3397         } /* for basic block */
3398
3399         dseg_createlinenumbertable(cd);
3400
3401         /* generate stubs */
3402
3403         emit_exception_stubs(jd);
3404         emit_patcher_stubs(jd);
3405 #if 0
3406         emit_replacement_stubs(jd);
3407 #endif
3408
3409         codegen_finish(jd);
3410
3411         /* everything's ok */
3412
3413         return true;
3414 }
3415
3416
3417 /* createcompilerstub **********************************************************
3418
3419    Creates a stub routine which calls the compiler.
3420
3421 *******************************************************************************/
3422
3423 #define COMPILERSTUB_DATASIZE    3 * SIZEOF_VOID_P
3424 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
3425
3426 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
3427
3428
3429 u1 *createcompilerstub(methodinfo *m)
3430 {
3431         u1          *s;                     /* memory to hold the stub            */
3432         ptrint      *d;
3433         codeinfo    *code;
3434         codegendata *cd;
3435         s4           dumpsize;
3436
3437         s = CNEW(u1, COMPILERSTUB_SIZE);
3438
3439         /* set data pointer and code pointer */
3440
3441         d = (ptrint *) s;
3442         s = s + COMPILERSTUB_DATASIZE;
3443
3444         /* mark start of dump memory area */
3445
3446         dumpsize = dump_size();
3447
3448         cd = DNEW(codegendata);
3449         cd->mcodeptr = s;
3450
3451         /* Store the codeinfo pointer in the same place as in the
3452            methodheader for compiled methods. */
3453
3454         code = code_codeinfo_new(m);
3455
3456         d[0] = (ptrint) asm_call_jit_compiler;
3457         d[1] = (ptrint) m;
3458         d[2] = (ptrint) code;
3459
3460         /* code for the stub */
3461
3462         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
3463         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
3464         M_JMP(REG_ITMP3);
3465
3466 #if defined(ENABLE_STATISTICS)
3467         if (opt_stat)
3468                 count_cstub_len += COMPILERSTUB_SIZE;
3469 #endif
3470
3471         /* release dump area */
3472
3473         dump_release(dumpsize);
3474
3475         return s;
3476 }
3477
3478
3479 /* createnativestub ************************************************************
3480
3481    Creates a stub routine which calls a native method.
3482
3483 *******************************************************************************/
3484
3485 u1 *createnativestub(functionptr f, jitdata *jd, methoddesc *nmd)
3486 {
3487         methodinfo   *m;
3488         codeinfo     *code;
3489         codegendata  *cd;
3490         registerdata *rd;
3491         methoddesc   *md;
3492         s4            nativeparams;
3493         s4            i, j;                 /* count variables                    */
3494         s4            t;
3495         s4            s1, s2;
3496
3497         /* get required compiler data */
3498
3499         m    = jd->m;
3500         code = jd->code;
3501         cd   = jd->cd;
3502         rd   = jd->rd;
3503
3504         /* initialize variables */
3505
3506         md = m->parseddesc;
3507         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
3508
3509         /* calculate stack frame size */
3510
3511         cd->stackframesize =
3512                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
3513                 sizeof(localref_table) / SIZEOF_VOID_P +
3514                 INT_ARG_CNT + FLT_ARG_CNT +
3515                 1 +                       /* functionptr, TODO: store in data segment */
3516                 nmd->memuse;
3517
3518         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
3519
3520         /* create method header */
3521
3522         (void) dseg_addaddress(cd, code);                      /* CodeinfoPointer */
3523         (void) dseg_adds4(cd, cd->stackframesize * 8);         /* FrameSize       */
3524         (void) dseg_adds4(cd, 0);                              /* IsSync          */
3525         (void) dseg_adds4(cd, 0);                              /* IsLeaf          */
3526         (void) dseg_adds4(cd, 0);                              /* IntSave         */
3527         (void) dseg_adds4(cd, 0);                              /* FltSave         */
3528         (void) dseg_addlinenumbertablesize(cd);
3529         (void) dseg_adds4(cd, 0);                              /* ExTableSize     */
3530
3531         /* generate native method profiling code */
3532
3533         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3534                 /* count frequency */
3535
3536                 M_MOV_IMM(code, REG_ITMP3);
3537                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3538         }
3539
3540         /* generate stub code */
3541
3542         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3543
3544 #if !defined(NDEBUG)
3545         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3546                 emit_verbosecall_enter(jd);
3547 #endif
3548
3549         /* get function address (this must happen before the stackframeinfo) */
3550
3551 #if !defined(WITH_STATIC_CLASSPATH)
3552         if (f == NULL) {
3553                 codegen_addpatchref(cd, PATCHER_resolve_native, m, 0);
3554
3555                 if (opt_showdisassemble) {
3556                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3557                 }
3558         }
3559 #endif
3560
3561         M_MOV_IMM(f, REG_ITMP3);
3562
3563
3564         /* save integer and float argument registers */
3565
3566         for (i = 0, j = 0; i < md->paramcount; i++) {
3567                 if (!md->params[i].inmemory) {
3568                         s1 = md->params[i].regoff;
3569
3570                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3571                                 M_LST(rd->argintregs[s1], REG_SP, j * 8);
3572                         else
3573                                 M_DST(rd->argfltregs[s1], REG_SP, j * 8);
3574
3575                         j++;
3576                 }
3577         }
3578
3579         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3580
3581         /* create dynamic stack info */
3582
3583         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3584         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3585         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3586         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3587         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3588         M_CALL(REG_ITMP1);
3589
3590         /* restore integer and float argument registers */
3591
3592         for (i = 0, j = 0; i < md->paramcount; i++) {
3593                 if (!md->params[i].inmemory) {
3594                         s1 = md->params[i].regoff;
3595
3596                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3597                                 M_LLD(rd->argintregs[s1], REG_SP, j * 8);
3598                         else
3599                                 M_DLD(rd->argfltregs[s1], REG_SP, j * 8);
3600
3601                         j++;
3602                 }
3603         }
3604
3605         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3606
3607
3608         /* copy or spill arguments to new locations */
3609
3610         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3611                 t = md->paramtypes[i].type;
3612
3613                 if (IS_INT_LNG_TYPE(t)) {
3614                         if (!md->params[i].inmemory) {
3615                                 s1 = rd->argintregs[md->params[i].regoff];
3616
3617                                 if (!nmd->params[j].inmemory) {
3618                                         s2 = rd->argintregs[nmd->params[j].regoff];
3619                                         M_INTMOVE(s1, s2);
3620
3621                                 } else {
3622                                         s2 = nmd->params[j].regoff;
3623                                         M_LST(s1, REG_SP, s2 * 8);
3624                                 }
3625
3626                         } else {
3627                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3628                                 s2 = nmd->params[j].regoff;
3629                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
3630                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
3631                         }
3632
3633                 } else {
3634                         /* We only copy spilled float arguments, as the float argument    */
3635                         /* registers keep unchanged.                                      */
3636
3637                         if (md->params[i].inmemory) {
3638                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3639                                 s2 = nmd->params[j].regoff;
3640
3641                                 if (IS_2_WORD_TYPE(t)) {
3642                                         M_DLD(REG_FTMP1, REG_SP, s1 * 8);
3643                                         M_DST(REG_FTMP1, REG_SP, s2 * 8);
3644                                 } else {
3645                                         M_FLD(REG_FTMP1, REG_SP, s1 * 8);
3646                                         M_FST(REG_FTMP1, REG_SP, s2 * 8);
3647                                 }
3648                         }
3649                 }
3650         }
3651
3652         /* put class into second argument register */
3653
3654         if (m->flags & ACC_STATIC)
3655                 M_MOV_IMM(m->class, REG_A1);
3656
3657         /* put env into first argument register */
3658
3659         M_MOV_IMM(_Jv_env, REG_A0);
3660
3661         /* do the native function call */
3662
3663         M_CALL(REG_ITMP3);
3664
3665         /* save return value */
3666
3667         if (md->returntype.type != TYPE_VOID) {
3668                 if (IS_INT_LNG_TYPE(md->returntype.type))
3669                         M_LST(REG_RESULT, REG_SP, 0 * 8);
3670                 else
3671                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
3672         }
3673
3674 #if !defined(NDEBUG)
3675         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3676                 emit_verbosecall_exit(jd);
3677 #endif
3678
3679         /* remove native stackframe info */
3680
3681         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3682         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3683         M_CALL(REG_ITMP1);
3684         M_MOV(REG_RESULT, REG_ITMP3);
3685
3686         /* restore return value */
3687
3688         if (md->returntype.type != TYPE_VOID) {
3689                 if (IS_INT_LNG_TYPE(md->returntype.type))
3690                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
3691                 else
3692                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3693         }
3694
3695         /* remove stackframe */
3696
3697         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3698
3699         /* test for exception */
3700
3701         M_TEST(REG_ITMP3);
3702         M_BNE(1);
3703         M_RET;
3704
3705         /* handle exception */
3706
3707         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3708         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3709         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3710
3711         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3712         M_JMP(REG_ITMP3);
3713
3714         /* generate patcher stubs */
3715
3716         emit_patcher_stubs(jd);
3717
3718         codegen_finish(jd);
3719
3720         return code->entrypoint;
3721 }
3722
3723
3724 /*
3725  * These are local overrides for various environment variables in Emacs.
3726  * Please do not remove this and leave it at the end of the file, where
3727  * Emacs will automagically detect them.
3728  * ---------------------------------------------------------------------
3729  * Local variables:
3730  * mode: c
3731  * indent-tabs-mode: t
3732  * c-basic-offset: 4
3733  * tab-width: 4
3734  * End:
3735  * vim:noexpandtab:sw=4:ts=4:
3736  */