src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29             Christian Ullrich
  30             Edwin Steiner
  31
  32    $Id: codegen.c 6047 2006-11-22 21:19:38Z twisti $
  33
  34 */
  35
  36
  37 #include "config.h"
  38
  39 #include <assert.h>
  40 #include <stdio.h>
  41
  42 #include "vm/types.h"
  43
  44 #include "md-abi.h"
  45
  46 #include "vm/jit/x86_64/arch.h"
  47 #include "vm/jit/x86_64/codegen.h"
  48 #include "vm/jit/x86_64/emit.h"
  49
  50 #include "mm/memory.h"
  51 #include "native/jni.h"
  52 #include "native/native.h"
  53
  54 #if defined(ENABLE_THREADS)
  55 # include "threads/native/lock.h"
  56 #endif
  57
  58 #include "vm/builtin.h"
  59 #include "vm/exceptions.h"
  60 #include "vm/global.h"
  61 #include "vm/loader.h"
  62 #include "vm/options.h"
  63 #include "vm/statistics.h"
  64 #include "vm/stringlocal.h"
  65 #include "vm/vm.h"
  66 #include "vm/jit/asmpart.h"
  67 #include "vm/jit/codegen-common.h"
  68 #include "vm/jit/dseg.h"
  69 #include "vm/jit/emit-common.h"
  70 #include "vm/jit/jit.h"
  71 #include "vm/jit/methodheader.h"
  72 #include "vm/jit/parse.h"
  73 #include "vm/jit/patcher.h"
  74 #include "vm/jit/reg.h"
  75 #include "vm/jit/replace.h"
  76
  77 #if defined(ENABLE_LSRA)
  78 # include "vm/jit/allocator/lsra.h"
  79 #endif
  80
  81
  82 /* codegen *********************************************************************
  83
  84    Generates machine code.
  85
  86 *******************************************************************************/
  87
  88 bool codegen(jitdata *jd)
  89 {
  90         methodinfo         *m;
  91         codeinfo           *code;
  92         codegendata        *cd;
  93         registerdata       *rd;
  94         s4                  len, s1, s2, s3, d, disp;
  95         u2                  currentline;
  96         ptrint              a;
  97         varinfo            *var, *var1, *var2, *dst;
  98         basicblock         *bptr;
  99         instruction        *iptr;
 100         exception_entry    *ex;
 101         constant_classref  *cr;
 102         unresolved_class   *uc;
 103         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 104         unresolved_method  *um;
 105         builtintable_entry *bte;
 106         methoddesc         *md;
 107         fieldinfo          *fi;
 108         unresolved_field   *uf;
 109         s4                  fieldtype;
 110         rplpoint           *replacementpoint;
 111         s4                 varindex;
 112
 113         /* get required compiler data */
 114
 115         m    = jd->m;
 116         code = jd->code;
 117         cd   = jd->cd;
 118         rd   = jd->rd;
 119
 120         /* prevent compiler warnings */
 121
 122         d   = 0;
 123         lm  = NULL;
 124         um  = NULL;
 125         bte = NULL;
 126
 127         {
 128         s4 i, p, t, l;
 129         s4 savedregs_num;
 130
 131         savedregs_num = 0;
 132
 133         /* space to save used callee saved registers */
 134
 135         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 136         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 137
 138         cd->stackframesize = rd->memuse + savedregs_num;
 139
 140 #if defined(ENABLE_THREADS)
 141         /* space to save argument of monitor_enter */
 142
 143         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 144                 cd->stackframesize++;
 145 #endif
 146
 147         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 148            native code e.g. libc or jni (alignment problems with
 149            movaps). */
 150
 151         if (!jd->isleafmethod || opt_verbosecall)
 152                 cd->stackframesize |= 0x1;
 153
 154         /* create method header */
 155
 156         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
 157         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
 158
 159 #if defined(ENABLE_THREADS)
 160         /* IsSync contains the offset relative to the stack pointer for the
 161            argument of monitor_exit used in the exception handler. Since the
 162            offset could be zero and give a wrong meaning of the flag it is
 163            offset by one.
 164         */
 165
 166         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 167                 (void) dseg_add_unique_s4(cd, (rd->memuse + 1) * 8); /* IsSync        */
 168         else
 169 #endif
 170                 (void) dseg_add_unique_s4(cd, 0);                  /* IsSync          */
 171
 172         (void) dseg_add_unique_s4(cd, jd->isleafmethod);               /* IsLeaf  */
 173         (void) dseg_add_unique_s4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave */
 174         (void) dseg_add_unique_s4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave */
 175
 176         (void) dseg_addlinenumbertablesize(cd);
 177
 178         (void) dseg_add_unique_s4(cd, jd->exceptiontablelength); /* ExTableSize   */
 179
 180         /* create exception table */
 181
 182         for (ex = jd->exceptiontable; ex != NULL; ex = ex->down) {
 183                 dseg_add_target(cd, ex->start);
 184                 dseg_add_target(cd, ex->end);
 185                 dseg_add_target(cd, ex->handler);
 186                 (void) dseg_add_unique_address(cd, ex->catchtype.any);
 187         }
 188
 189         /* generate method profiling code */
 190
 191         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 192                 /* count frequency */
 193
 194                 M_MOV_IMM(code, REG_ITMP3);
 195                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 196
 197                 PROFILE_CYCLE_START;
 198         }
 199
 200         /* create stack frame (if necessary) */
 201
 202         if (cd->stackframesize)
 203                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 204
 205         /* save used callee saved registers */
 206
 207         p = cd->stackframesize;
 208         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 209                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 210         }
 211         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 212                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 213         }
 214
 215         /* take arguments out of register or stack frame */
 216
 217         md = m->parseddesc;
 218
 219         for (p = 0, l = 0; p < md->paramcount; p++) {
 220                 t = md->paramtypes[p].type;
 221
 222                 varindex = jd->local_map[l * 5 + t];
 223
 224                 l++;
 225                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 226                         l++;
 227
 228                 if (varindex == UNUSED)
 229                         continue;
 230
 231                 var = VAR(varindex);
 232
 233                 s1 = md->params[p].regoff;
 234
 235                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 236                         s2 = rd->argintregs[s1];
 237                         if (!md->params[p].inmemory) {           /* register arguments    */
 238                                 if (!IS_INMEMORY(var->flags)) {      /* reg arg -> register   */
 239                                         M_INTMOVE(s2, var->vv.regoff);
 240
 241                                 } else {                             /* reg arg -> spilled    */
 242                                     M_LST(s2, REG_SP, var->vv.regoff * 8);
 243                                 }
 244
 245                         } else {                                 /* stack arguments       */
 246                                 if (!IS_INMEMORY(var->flags)) {      /* stack arg -> register */
 247                                         /* + 8 for return address */
 248                                         M_LLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 249
 250                                 } else {                             /* stack arg -> spilled  */
 251                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 252                                 }
 253                         }
 254
 255                 } else {                                     /* floating args         */
 256                         if (!md->params[p].inmemory) {           /* register arguments    */
 257                                 s2 = rd->argfltregs[s1];
 258                                 if (!IS_INMEMORY(var->flags)) {      /* reg arg -> register   */
 259                                         M_FLTMOVE(s2, var->vv.regoff);
 260
 261                                 } else {                                         /* reg arg -> spilled    */
 262                                         M_DST(s2, REG_SP, var->vv.regoff * 8);
 263                                 }
 264
 265                         } else {                                 /* stack arguments       */
 266                                 if (!IS_INMEMORY(var->flags)) {      /* stack-arg -> register */
 267                                         M_DLD(var->vv.regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 268
 269                                 } else {
 270                                         var->vv.regoff = cd->stackframesize + s1 + 1;
 271                                 }
 272                         }
 273                 }
 274         }  /* end for */
 275
 276         /* save monitorenter argument */
 277
 278 #if defined(ENABLE_THREADS)
 279         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 280                 /* stack offset for monitor argument */
 281
 282                 s1 = rd->memuse;
 283
 284                 if (opt_verbosecall) {
 285                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 286
 287                         for (p = 0; p < INT_ARG_CNT; p++)
 288                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 289
 290                         for (p = 0; p < FLT_ARG_CNT; p++)
 291                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 292
 293                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 294                 }
 295
 296                 /* decide which monitor enter function to call */
 297
 298                 if (m->flags & ACC_STATIC) {
 299                         M_MOV_IMM(&m->class->object.header, REG_A0);
 300                 }
 301                 else {
 302                         M_TEST(REG_A0);
 303                         M_BEQ(0);
 304                         codegen_add_nullpointerexception_ref(cd);
 305                 }
 306
 307                 M_AST(REG_A0, REG_SP, s1 * 8);
 308                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 309                 M_CALL(REG_ITMP1);
 310
 311                 if (opt_verbosecall) {
 312                         for (p = 0; p < INT_ARG_CNT; p++)
 313                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 314
 315                         for (p = 0; p < FLT_ARG_CNT; p++)
 316                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 317
 318                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 319                 }
 320         }
 321 #endif
 322
 323 #if !defined(NDEBUG)
 324         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 325                 emit_verbosecall_enter(jd);
 326 #endif /* !defined(NDEBUG) */
 327
 328         }
 329
 330         /* end of header generation */
 331
 332         replacementpoint = jd->code->rplpoints;
 333
 334         /* walk through all basic blocks */
 335
 336         for (bptr = jd->basicblocks; bptr != NULL; bptr = bptr->next) {
 337
 338                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 339
 340                 if (bptr->flags >= BBREACHED) {
 341
 342                 /* branch resolving */
 343
 344                 codegen_resolve_branchrefs(cd, bptr);
 345
 346                 /* handle replacement points */
 347
 348 #if 0
 349                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 350                         replacementpoint->pc = (u1*)(ptrint)bptr->mpc; /* will be resolved later */
 351
 352                         replacementpoint++;
 353
 354                         assert(cd->lastmcodeptr <= cd->mcodeptr);
 355                         cd->lastmcodeptr = cd->mcodeptr + 5; /* 5 byte jmp patch */
 356                 }
 357 #endif
 358
 359                 /* copy interface registers to their destination */
 360
 361                 len = bptr->indepth;
 362                 MCODECHECK(512);
 363
 364                 /* generate basicblock profiling code */
 365
 366                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 367                         /* count frequency */
 368
 369                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 370                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 371
 372                         /* if this is an exception handler, start profiling again */
 373
 374                         if (bptr->type == BBTYPE_EXH)
 375                                 PROFILE_CYCLE_START;
 376                 }
 377
 378 #if defined(ENABLE_LSRA)
 379                 if (opt_lsra) {
 380                         while (len) {
 381                                 len--;
 382                                 src = bptr->invars[len];
 383                                 if ((len == bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 384                                         if (bptr->type == BBTYPE_EXH) {
 385 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 386                                                 if (!IS_INMEMORY(src->flags))
 387                                                         d= src->vv.regoff;
 388                                                 else
 389                                                         d=REG_ITMP1;
 390                                                 M_INTMOVE(REG_ITMP1, d);
 391                                                 emit_store(jd, NULL, src, d);
 392                                         }
 393                                 }
 394                         }
 395
 396                 } else {
 397 #endif
 398
 399                 while (len) {
 400                         len--;
 401                         var = VAR(bptr->invars[len]);
 402                         if ((len ==  bptr->indepth-1) && (bptr->type != BBTYPE_STD)) {
 403                                 if (bptr->type == BBTYPE_EXH) {
 404                                         d = codegen_reg_of_var(0, var, REG_ITMP1);
 405                                         M_INTMOVE(REG_ITMP1, d);
 406                                         emit_store(jd, NULL, var, d);
 407                                 }
 408                         }
 409                         else {
 410                                 assert((var->flags & INOUT));
 411                         }
 412                 }
 413 #if defined(ENABLE_LSRA)
 414                 }
 415 #endif
 416                 /* walk through all instructions */
 417
 418                 len = bptr->icount;
 419                 currentline = 0;
 420
 421                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 422                         if (iptr->line != currentline) {
 423                                 dseg_addlinenumber(cd, iptr->line);
 424                                 currentline = iptr->line;
 425                         }
 426
 427                         MCODECHECK(1024);                         /* 1KB should be enough */
 428
 429                 switch (iptr->opc) {
 430                 case ICMD_NOP:        /* ...  ==> ...                                 */
 431                 case ICMD_POP:        /* ..., value  ==> ...                          */
 432                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 433                 case ICMD_INLINE_START: /* internal ICMDs                         */
 434                 case ICMD_INLINE_END:
 435                         break;
 436
 437                 case ICMD_CHECKNULL:  /* ..., objectref  ==> ..., objectref           */
 438
 439                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 440                         M_TEST(s1);
 441                         M_BEQ(0);
 442                         codegen_add_nullpointerexception_ref(cd);
 443                         break;
 444
 445                 /* constant operations ************************************************/
 446
 447                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 448
 449                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 450                         ICONST(d, iptr->sx.val.i);
 451                         emit_store_dst(jd, iptr, d);
 452                         break;
 453
 454                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 455
 456                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 457                         LCONST(d, iptr->sx.val.l);
 458                         emit_store_dst(jd, iptr, d);
 459                         break;
 460
 461                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 462
 463                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 464                         disp = dseg_add_float(cd, iptr->sx.val.f);
 465                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 466                         emit_store_dst(jd, iptr, d);
 467                         break;
 468
 469                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 470
 471                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 472                         disp = dseg_add_double(cd, iptr->sx.val.d);
 473                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 474                         emit_store_dst(jd, iptr, d);
 475                         break;
 476
 477                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 478
 479                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 480
 481                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 482                                 cr = iptr->sx.val.c.ref;
 483
 484 /*                              PROFILE_CYCLE_STOP; */
 485
 486                                 codegen_add_patch_ref(cd, PATCHER_aconst, cr, 0);
 487
 488 /*                              PROFILE_CYCLE_START; */
 489
 490                                 M_MOV_IMM(NULL, d);
 491
 492                         } else {
 493                                 if (iptr->sx.val.anyptr == 0)
 494                                         M_CLR(d);
 495                                 else
 496                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 497                         }
 498                         emit_store_dst(jd, iptr, d);
 499                         break;
 500
 501
 502                 /* load/store/copy/move operations ************************************/
 503
 504                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 505                 case ICMD_ALOAD:      /* s1 = local variable                          */
 506                 case ICMD_LLOAD:
 507                 case ICMD_FLOAD:
 508                 case ICMD_DLOAD:
 509                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 510                 case ICMD_LSTORE:
 511                 case ICMD_FSTORE:
 512                 case ICMD_DSTORE:
 513                 case ICMD_COPY:
 514                 case ICMD_MOVE:
 515
 516                         emit_copy(jd, iptr, VAROP(iptr->s1), VAROP(iptr->dst));
 517                         break;
 518
 519                 case ICMD_ASTORE:
 520                         if (!(iptr->flags.bits & INS_FLAG_RETADDR))
 521                                 emit_copy(jd, iptr, VAROP(iptr->s1), VAROP(iptr->dst));
 522                         break;
 523
 524                 /* integer operations *************************************************/
 525
 526                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 527
 528                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 529                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 530                         M_INTMOVE(s1, d);
 531                         M_INEG(d);
 532                         emit_store_dst(jd, iptr, d);
 533                         break;
 534
 535                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 536
 537                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 538                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 539                         M_INTMOVE(s1, d);
 540                         M_LNEG(d);
 541                         emit_store_dst(jd, iptr, d);
 542                         break;
 543
 544                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 545
 546                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 547                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 548                         M_ISEXT(s1, d);
 549                         emit_store_dst(jd, iptr, d);
 550                         break;
 551
 552                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 553
 554                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 555                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 556                         M_IMOV(s1, d);
 557                         emit_store_dst(jd, iptr, d);
 558                         break;
 559
 560                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 561
 562                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 563                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 564                         M_BSEXT(s1, d);
 565                         emit_store_dst(jd, iptr, d);
 566                         break;
 567
 568                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 569
 570                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 571                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 572                         M_CZEXT(s1, d);
 573                         emit_store_dst(jd, iptr, d);
 574                         break;
 575
 576                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 577
 578                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 579                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 580                         M_SSEXT(s1, d);
 581                         emit_store_dst(jd, iptr, d);
 582                         break;
 583
 584
 585                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 586
 587                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 588                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 589                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 590                         if (s2 == d)
 591                                 M_IADD(s1, d);
 592                         else {
 593                                 M_INTMOVE(s1, d);
 594                                 M_IADD(s2, d);
 595                         }
 596                         emit_store_dst(jd, iptr, d);
 597                         break;
 598
 599                 case ICMD_IINC:
 600                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 601                                       /* sx.val.i = constant                             */
 602
 603                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 604                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 605
 606                         /* Using inc and dec is not faster than add (tested with
 607                            sieve). */
 608
 609                         M_INTMOVE(s1, d);
 610                         M_IADD_IMM(iptr->sx.val.i, d);
 611                         emit_store_dst(jd, iptr, d);
 612                         break;
 613
 614                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 615
 616                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 617                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 618                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 619                         if (s2 == d)
 620                                 M_LADD(s1, d);
 621                         else {
 622                                 M_INTMOVE(s1, d);
 623                                 M_LADD(s2, d);
 624                         }
 625                         emit_store_dst(jd, iptr, d);
 626                         break;
 627
 628                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 629                                       /* sx.val.l = constant                             */
 630
 631                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 632                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 633                         M_INTMOVE(s1, d);
 634                         if (IS_IMM32(iptr->sx.val.l))
 635                                 M_LADD_IMM(iptr->sx.val.l, d);
 636                         else {
 637                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 638                                 M_LADD(REG_ITMP2, d);
 639                         }
 640                         emit_store_dst(jd, iptr, d);
 641                         break;
 642
 643                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 644
 645                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 646                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 647                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 648                         if (s2 == d) {
 649                                 M_INTMOVE(s1, REG_ITMP1);
 650                                 M_ISUB(s2, REG_ITMP1);
 651                                 M_INTMOVE(REG_ITMP1, d);
 652                         } else {
 653                                 M_INTMOVE(s1, d);
 654                                 M_ISUB(s2, d);
 655                         }
 656                         emit_store_dst(jd, iptr, d);
 657                         break;
 658
 659                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 660                                       /* sx.val.i = constant                             */
 661
 662                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 663                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 664                         M_INTMOVE(s1, d);
 665                         M_ISUB_IMM(iptr->sx.val.i, d);
 666                         emit_store_dst(jd, iptr, d);
 667                         break;
 668
 669                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 670
 671                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 672                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 673                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 674                         if (s2 == d) {
 675                                 M_INTMOVE(s1, REG_ITMP1);
 676                                 M_LSUB(s2, REG_ITMP1);
 677                                 M_INTMOVE(REG_ITMP1, d);
 678                         } else {
 679                                 M_INTMOVE(s1, d);
 680                                 M_LSUB(s2, d);
 681                         }
 682                         emit_store_dst(jd, iptr, d);
 683                         break;
 684
 685                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 686                                       /* sx.val.l = constant                             */
 687
 688                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 689                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 690                         M_INTMOVE(s1, d);
 691                         if (IS_IMM32(iptr->sx.val.l))
 692                                 M_LSUB_IMM(iptr->sx.val.l, d);
 693                         else {
 694                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 695                                 M_LSUB(REG_ITMP2, d);
 696                         }
 697                         emit_store_dst(jd, iptr, d);
 698                         break;
 699
 700                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 701
 702                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 703                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 704                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 705                         if (s2 == d)
 706                                 M_IMUL(s1, d);
 707                         else {
 708                                 M_INTMOVE(s1, d);
 709                                 M_IMUL(s2, d);
 710                         }
 711                         emit_store_dst(jd, iptr, d);
 712                         break;
 713
 714                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 715                                       /* sx.val.i = constant                             */
 716
 717                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 718                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 719                         if (iptr->sx.val.i == 2) {
 720                                 M_INTMOVE(s1, d);
 721                                 M_ISLL_IMM(1, d);
 722                         } else
 723                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 724                         emit_store_dst(jd, iptr, d);
 725                         break;
 726
 727                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 728
 729                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 730                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 731                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 732                         if (s2 == d)
 733                                 M_LMUL(s1, d);
 734                         else {
 735                                 M_INTMOVE(s1, d);
 736                                 M_LMUL(s2, d);
 737                         }
 738                         emit_store_dst(jd, iptr, d);
 739                         break;
 740
 741                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 742                                       /* sx.val.l = constant                             */
 743
 744                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 745                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 746                         if (IS_IMM32(iptr->sx.val.l))
 747                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 748                         else {
 749                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 750                                 M_INTMOVE(s1, d);
 751                                 M_LMUL(REG_ITMP2, d);
 752                         }
 753                         emit_store_dst(jd, iptr, d);
 754                         break;
 755
 756                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 757
 758                         var1 = VAROP(iptr->s1);
 759                         var2 = VAROP(iptr->sx.s23.s2);
 760                         dst  = VAROP(iptr->dst);
 761
 762                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 763                         if (IS_INMEMORY(var1->flags))
 764                                 M_ILD(RAX, REG_SP, var1->vv.regoff * 8);
 765                         else
 766                                 M_INTMOVE(var1->vv.regoff, RAX);
 767
 768                         if (IS_INMEMORY(var2->flags))
 769                                 M_ILD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 770                         else
 771                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 772
 773                         if (checknull) {
 774                                 M_ITEST(REG_ITMP3);
 775                                 M_BEQ(0);
 776                                 codegen_add_arithmeticexception_ref(cd);
 777                         }
 778
 779                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
 780                         emit_jcc(cd, CC_NE, 4 + 6);
 781                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
 782                         emit_jcc(cd, CC_E, 3 + 1 + 3);                /* 6 bytes */
 783
 784                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 785                         emit_cltd(cd);
 786                         emit_idivl_reg(cd, REG_ITMP3);
 787
 788                         if (IS_INMEMORY(dst->flags)) {
 789                                 emit_mov_reg_membase(cd, RAX, REG_SP, dst->vv.regoff * 8);
 790                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
 791
 792                         } else {
 793                                 M_INTMOVE(RAX, dst->vv.regoff);
 794
 795                                 if (dst->vv.regoff != RDX) {
 796                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
 797                                 }
 798                         }
 799                         break;
 800
 801                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 802
 803                         var1 = VAROP(iptr->s1);
 804                         var2 = VAROP(iptr->sx.s23.s2);
 805                         dst  = VAROP(iptr->dst);
 806
 807                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 808                         if (IS_INMEMORY(var1->flags))
 809                                 M_ILD(RAX, REG_SP, var1->vv.regoff * 8);
 810                         else
 811                                 M_INTMOVE(var1->vv.regoff, RAX);
 812
 813                         if (IS_INMEMORY(var2->flags))
 814                                 M_ILD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 815                         else
 816                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 817
 818                         if (checknull) {
 819                                 M_ITEST(REG_ITMP3);
 820                                 M_BEQ(0);
 821                                 codegen_add_arithmeticexception_ref(cd);
 822                         }
 823
 824                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 825
 826                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
 827                         emit_jcc(cd, CC_NE, 2 + 4 + 6);
 828
 829
 830                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
 831                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
 832                         emit_jcc(cd, CC_E, 1 + 3);                    /* 6 bytes */
 833
 834                         emit_cltd(cd);
 835                         emit_idivl_reg(cd, REG_ITMP3);
 836
 837                         if (IS_INMEMORY(dst->flags)) {
 838                                 emit_mov_reg_membase(cd, RDX, REG_SP, dst->vv.regoff * 8);
 839                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
 840
 841                         } else {
 842                                 M_INTMOVE(RDX, dst->vv.regoff);
 843
 844                                 if (dst->vv.regoff != RDX) {
 845                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
 846                                 }
 847                         }
 848                         break;
 849
 850                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 851                                       /* sx.val.i = constant                             */
 852
 853                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 854                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 855                         M_INTMOVE(s1, REG_ITMP1);
 856                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 857                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 858                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 859                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 860                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 861                         emit_store_dst(jd, iptr, d);
 862                         break;
 863
 864                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
 865                                       /* sx.val.i = constant                             */
 866
 867                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 868                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 869                         M_INTMOVE(s1, REG_ITMP1);
 870                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 871                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
 872                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
 873                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
 874                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
 875                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 876                         emit_store_dst(jd, iptr, d);
 877                         break;
 878
 879
 880                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 881
 882                         var1 = VAROP(iptr->s1);
 883                         var2 = VAROP(iptr->sx.s23.s2);
 884                         dst  = VAROP(iptr->dst);
 885
 886                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 887
 888                 if (IS_INMEMORY(var1->flags))
 889                                 M_LLD(RAX, REG_SP, var1->vv.regoff * 8);
 890                         else
 891                                 M_INTMOVE(var1->vv.regoff, RAX);
 892
 893                         if (IS_INMEMORY(var2->flags))
 894                                 M_LLD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 895                         else
 896                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 897
 898                         if (checknull) {
 899                                 M_TEST(REG_ITMP3);
 900                                 M_BEQ(0);
 901                                 codegen_add_arithmeticexception_ref(cd);
 902                         }
 903
 904                         /* check as described in jvm spec */
 905                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 906                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
 907                         M_BNE(4 + 6);
 908                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
 909                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
 910
 911                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 912                         emit_cqto(cd);
 913                         emit_idiv_reg(cd, REG_ITMP3);
 914
 915                         if (IS_INMEMORY(dst->flags)) {
 916                                 M_LST(RAX, REG_SP, dst->vv.regoff * 8);
 917                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
 918
 919                         } else {
 920                                 M_INTMOVE(RAX, dst->vv.regoff);
 921
 922                                 if (dst->vv.regoff != RDX) {
 923                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
 924                                 }
 925                         }
 926                         break;
 927
 928                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 929
 930                         var1 = VAROP(iptr->s1);
 931                         var2 = VAROP(iptr->sx.s23.s2);
 932                         dst  = VAROP(iptr->dst);
 933
 934                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 935
 936                         if (IS_INMEMORY(var1->flags))
 937                                 M_LLD(REG_ITMP1, REG_SP, var1->vv.regoff * 8);
 938                         else
 939                                 M_INTMOVE(var1->vv.regoff, REG_ITMP1);
 940
 941                         if (IS_INMEMORY(var2->flags))
 942                                 M_LLD(REG_ITMP3, REG_SP, var2->vv.regoff * 8);
 943                         else
 944                                 M_INTMOVE(var2->vv.regoff, REG_ITMP3);
 945                         /*
 946                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 947                         M_INTMOVE(s1, REG_ITMP1);
 948                         s2 = emit_load_s2(jd, iptr, REG_ITMP3);
 949                         M_INTMOVE(s2, REG_ITMP3);
 950                         */
 951                         if (checknull) {
 952                                 M_ITEST(REG_ITMP3);
 953                                 M_BEQ(0);
 954                                 codegen_add_arithmeticexception_ref(cd);
 955                         }
 956
 957                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 958
 959                         /* check as described in jvm spec */
 960                         disp = dseg_add_s8(cd, 0x8000000000000000LL);
 961                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
 962                         M_BNE(3 + 4 + 6);
 963
 964 #if 0
 965                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
 966 #endif
 967                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
 968                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
 969                         M_BEQ(2 + 3);                                          /* 6 bytes */
 970
 971                         emit_cqto(cd);
 972                         emit_idiv_reg(cd, REG_ITMP3);
 973
 974                         if (IS_INMEMORY(dst->flags)) {
 975                                 M_LST(RDX, REG_SP, dst->vv.regoff * 8);
 976                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
 977
 978                         } else {
 979                                 M_INTMOVE(RDX, dst->vv.regoff);
 980
 981                                 if (dst->vv.regoff != RDX) {
 982                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
 983                                 }
 984                         }
 985                         break;
 986
 987                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
 988                                       /* sx.val.i = constant                             */
 989
 990                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 991                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 992                         M_INTMOVE(s1, REG_ITMP1);
 993                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
 994                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
 995                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
 996                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
 997                         emit_mov_reg_reg(cd, REG_ITMP1, d);
 998                         emit_store_dst(jd, iptr, d);
 999                         break;
1000
1001                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1002                                       /* sx.val.l = constant                             */
1003
1004                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1005                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1006                         M_INTMOVE(s1, REG_ITMP1);
1007                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1008                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
1009                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
1010                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
1011                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
1012                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1013                         emit_store_dst(jd, iptr, d);
1014                         break;
1015
1016                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1017
1018                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1019                         emit_ishift(jd, SHIFT_SHL, iptr);
1020                         break;
1021
1022                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1023                                       /* sx.val.i = constant                             */
1024
1025                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1026                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1027                         M_INTMOVE(s1, d);
1028                         M_ISLL_IMM(iptr->sx.val.i, d);
1029                         emit_store_dst(jd, iptr, d);
1030                         break;
1031
1032                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1033
1034                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1035                         emit_ishift(jd, SHIFT_SAR, iptr);
1036                         break;
1037
1038                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1039                                       /* sx.val.i = constant                             */
1040
1041                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1042                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1043                         M_INTMOVE(s1, d);
1044                         M_ISRA_IMM(iptr->sx.val.i, d);
1045                         emit_store_dst(jd, iptr, d);
1046                         break;
1047
1048                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1049
1050                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1051                         emit_ishift(jd, SHIFT_SHR, iptr);
1052                         break;
1053
1054                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1055                                       /* sx.val.i = constant                             */
1056
1057                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1058                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1059                         M_INTMOVE(s1, d);
1060                         M_ISRL_IMM(iptr->sx.val.i, d);
1061                         emit_store_dst(jd, iptr, d);
1062                         break;
1063
1064                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1065
1066                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1067                         emit_lshift(jd, SHIFT_SHL, iptr);
1068                         break;
1069
1070         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1071                                           /* sx.val.i = constant                             */
1072
1073                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1074                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1075                         M_INTMOVE(s1, d);
1076                         M_LSLL_IMM(iptr->sx.val.i, d);
1077                         emit_store_dst(jd, iptr, d);
1078                         break;
1079
1080                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1081
1082                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1083                         emit_lshift(jd, SHIFT_SAR, iptr);
1084                         break;
1085
1086                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1087                                       /* sx.val.i = constant                             */
1088
1089                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1090                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1091                         M_INTMOVE(s1, d);
1092                         M_LSRA_IMM(iptr->sx.val.i, d);
1093                         emit_store_dst(jd, iptr, d);
1094                         break;
1095
1096                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1097
1098                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1099                         emit_lshift(jd, SHIFT_SHR, iptr);
1100                         break;
1101
1102                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1103                                       /* sx.val.l = constant                             */
1104
1105                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1106                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1107                         M_INTMOVE(s1, d);
1108                         M_LSRL_IMM(iptr->sx.val.i, d);
1109                         emit_store_dst(jd, iptr, d);
1110                         break;
1111
1112                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1113
1114                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1115                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1116                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1117                         if (s2 == d)
1118                                 M_IAND(s1, d);
1119                         else {
1120                                 M_INTMOVE(s1, d);
1121                                 M_IAND(s2, d);
1122                         }
1123                         emit_store_dst(jd, iptr, d);
1124                         break;
1125
1126                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1127                                       /* sx.val.i = constant                             */
1128
1129                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1130                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1131                         M_INTMOVE(s1, d);
1132                         M_IAND_IMM(iptr->sx.val.i, d);
1133                         emit_store_dst(jd, iptr, d);
1134                         break;
1135
1136                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1137
1138                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1139                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1140                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1141                         if (s2 == d)
1142                                 M_LAND(s1, d);
1143                         else {
1144                                 M_INTMOVE(s1, d);
1145                                 M_LAND(s2, d);
1146                         }
1147                         emit_store_dst(jd, iptr, d);
1148                         break;
1149
1150                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1151                                       /* sx.val.l = constant                             */
1152
1153                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1154                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1155                         M_INTMOVE(s1, d);
1156                         if (IS_IMM32(iptr->sx.val.l))
1157                                 M_LAND_IMM(iptr->sx.val.l, d);
1158                         else {
1159                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1160                                 M_LAND(REG_ITMP2, d);
1161                         }
1162                         emit_store_dst(jd, iptr, d);
1163                         break;
1164
1165                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1166
1167                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1168                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1169                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1170                         if (s2 == d)
1171                                 M_IOR(s1, d);
1172                         else {
1173                                 M_INTMOVE(s1, d);
1174                                 M_IOR(s2, d);
1175                         }
1176                         emit_store_dst(jd, iptr, d);
1177                         break;
1178
1179                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1180                                       /* sx.val.i = constant                             */
1181
1182                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1183                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1184                         M_INTMOVE(s1, d);
1185                         M_IOR_IMM(iptr->sx.val.i, d);
1186                         emit_store_dst(jd, iptr, d);
1187                         break;
1188
1189                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1190
1191                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1192                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1193                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1194                         if (s2 == d)
1195                                 M_LOR(s1, d);
1196                         else {
1197                                 M_INTMOVE(s1, d);
1198                                 M_LOR(s2, d);
1199                         }
1200                         emit_store_dst(jd, iptr, d);
1201                         break;
1202
1203                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1204                                       /* sx.val.l = constant                             */
1205
1206                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1207                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1208                         M_INTMOVE(s1, d);
1209                         if (IS_IMM32(iptr->sx.val.l))
1210                                 M_LOR_IMM(iptr->sx.val.l, d);
1211                         else {
1212                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1213                                 M_LOR(REG_ITMP2, d);
1214                         }
1215                         emit_store_dst(jd, iptr, d);
1216                         break;
1217
1218                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1219
1220                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1221                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1222                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1223                         if (s2 == d)
1224                                 M_IXOR(s1, d);
1225                         else {
1226                                 M_INTMOVE(s1, d);
1227                                 M_IXOR(s2, d);
1228                         }
1229                         emit_store_dst(jd, iptr, d);
1230                         break;
1231
1232                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1233                                       /* sx.val.i = constant                             */
1234
1235                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1236                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1237                         M_INTMOVE(s1, d);
1238                         M_IXOR_IMM(iptr->sx.val.i, d);
1239                         emit_store_dst(jd, iptr, d);
1240                         break;
1241
1242                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1243
1244                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1245                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1246                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1247                         if (s2 == d)
1248                                 M_LXOR(s1, d);
1249                         else {
1250                                 M_INTMOVE(s1, d);
1251                                 M_LXOR(s2, d);
1252                         }
1253                         emit_store_dst(jd, iptr, d);
1254                         break;
1255
1256                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1257                                       /* sx.val.l = constant                             */
1258
1259                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1260                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1261                         M_INTMOVE(s1, d);
1262                         if (IS_IMM32(iptr->sx.val.l))
1263                                 M_LXOR_IMM(iptr->sx.val.l, d);
1264                         else {
1265                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1266                                 M_LXOR(REG_ITMP2, d);
1267                         }
1268                         emit_store_dst(jd, iptr, d);
1269                         break;
1270
1271
1272                 /* floating operations ************************************************/
1273
1274                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1275
1276                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1277                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1278                         disp = dseg_add_s4(cd, 0x80000000);
1279                         M_FLTMOVE(s1, d);
1280                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1281                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1282                         emit_store_dst(jd, iptr, d);
1283                         break;
1284
1285                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1286
1287                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1288                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1289                         disp = dseg_add_s8(cd, 0x8000000000000000);
1290                         M_FLTMOVE(s1, d);
1291                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1292                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1293                         emit_store_dst(jd, iptr, d);
1294                         break;
1295
1296                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1297
1298                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1299                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1300                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1301                         if (s2 == d)
1302                                 M_FADD(s1, d);
1303                         else {
1304                                 M_FLTMOVE(s1, d);
1305                                 M_FADD(s2, d);
1306                         }
1307                         emit_store_dst(jd, iptr, d);
1308                         break;
1309
1310                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1311
1312                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1313                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1314                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1315                         if (s2 == d)
1316                                 M_DADD(s1, d);
1317                         else {
1318                                 M_FLTMOVE(s1, d);
1319                                 M_DADD(s2, d);
1320                         }
1321                         emit_store_dst(jd, iptr, d);
1322                         break;
1323
1324                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1325
1326                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1327                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1328                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1329                         if (s2 == d) {
1330                                 M_FLTMOVE(s2, REG_FTMP2);
1331                                 s2 = REG_FTMP2;
1332                         }
1333                         M_FLTMOVE(s1, d);
1334                         M_FSUB(s2, d);
1335                         emit_store_dst(jd, iptr, d);
1336                         break;
1337
1338                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1339
1340                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1341                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1342                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1343                         if (s2 == d) {
1344                                 M_FLTMOVE(s2, REG_FTMP2);
1345                                 s2 = REG_FTMP2;
1346                         }
1347                         M_FLTMOVE(s1, d);
1348                         M_DSUB(s2, d);
1349                         emit_store_dst(jd, iptr, d);
1350                         break;
1351
1352                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1353
1354                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1355                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1356                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1357                         if (s2 == d)
1358                                 M_FMUL(s1, d);
1359                         else {
1360                                 M_FLTMOVE(s1, d);
1361                                 M_FMUL(s2, d);
1362                         }
1363                         emit_store_dst(jd, iptr, d);
1364                         break;
1365
1366                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1367
1368                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1369                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1370                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1371                         if (s2 == d)
1372                                 M_DMUL(s1, d);
1373                         else {
1374                                 M_FLTMOVE(s1, d);
1375                                 M_DMUL(s2, d);
1376                         }
1377                         emit_store_dst(jd, iptr, d);
1378                         break;
1379
1380                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1381
1382                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1383                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1384                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1385                         if (s2 == d) {
1386                                 M_FLTMOVE(s2, REG_FTMP2);
1387                                 s2 = REG_FTMP2;
1388                         }
1389                         M_FLTMOVE(s1, d);
1390                         M_FDIV(s2, d);
1391                         emit_store_dst(jd, iptr, d);
1392                         break;
1393
1394                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1395
1396                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1397                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1398                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1399                         if (s2 == d) {
1400                                 M_FLTMOVE(s2, REG_FTMP2);
1401                                 s2 = REG_FTMP2;
1402                         }
1403                         M_FLTMOVE(s1, d);
1404                         M_DDIV(s2, d);
1405                         emit_store_dst(jd, iptr, d);
1406                         break;
1407
1408                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1409
1410                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1411                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1412                         M_CVTIF(s1, d);
1413                         emit_store_dst(jd, iptr, d);
1414                         break;
1415
1416                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1417
1418                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1419                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1420                         M_CVTID(s1, d);
1421                         emit_store_dst(jd, iptr, d);
1422                         break;
1423
1424                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1425
1426                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1427                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1428                         M_CVTLF(s1, d);
1429                         emit_store_dst(jd, iptr, d);
1430                         break;
1431
1432                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1433
1434                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1435                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1436                         M_CVTLD(s1, d);
1437                         emit_store_dst(jd, iptr, d);
1438                         break;
1439
1440                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1441
1442                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1443                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1444                         M_CVTFI(s1, d);
1445                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1446                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1447                                 ((REG_RESULT == d) ? 0 : 3);
1448                         M_BNE(disp);
1449                         M_FLTMOVE(s1, REG_FTMP1);
1450                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1451                         M_CALL(REG_ITMP2);
1452                         M_INTMOVE(REG_RESULT, d);
1453                         emit_store_dst(jd, iptr, d);
1454                         break;
1455
1456                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1457
1458                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1459                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1460                         M_CVTDI(s1, d);
1461                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1462                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1463                                 ((REG_RESULT == d) ? 0 : 3);
1464                         M_BNE(disp);
1465                         M_FLTMOVE(s1, REG_FTMP1);
1466                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1467                         M_CALL(REG_ITMP2);
1468                         M_INTMOVE(REG_RESULT, d);
1469                         emit_store_dst(jd, iptr, d);
1470                         break;
1471
1472                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1473
1474                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1475                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1476                         M_CVTFL(s1, d);
1477                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1478                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1479                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1480                                 ((REG_RESULT == d) ? 0 : 3);
1481                         M_BNE(disp);
1482                         M_FLTMOVE(s1, REG_FTMP1);
1483                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1484                         M_CALL(REG_ITMP2);
1485                         M_INTMOVE(REG_RESULT, d);
1486                         emit_store_dst(jd, iptr, d);
1487                         break;
1488
1489                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1490
1491                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1492                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1493                         M_CVTDL(s1, d);
1494                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1495                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1496                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1497                                 ((REG_RESULT == d) ? 0 : 3);
1498                         M_BNE(disp);
1499                         M_FLTMOVE(s1, REG_FTMP1);
1500                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1501                         M_CALL(REG_ITMP2);
1502                         M_INTMOVE(REG_RESULT, d);
1503                         emit_store_dst(jd, iptr, d);
1504                         break;
1505
1506                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1507
1508                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1509                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1510                         M_CVTFD(s1, d);
1511                         emit_store_dst(jd, iptr, d);
1512                         break;
1513
1514                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1515
1516                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1517                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1518                         M_CVTDF(s1, d);
1519                         emit_store_dst(jd, iptr, d);
1520                         break;
1521
1522                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1523                                           /* == => 0, < => 1, > => -1 */
1524
1525                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1526                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1527                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1528                         M_CLR(d);
1529                         M_MOV_IMM(1, REG_ITMP1);
1530                         M_MOV_IMM(-1, REG_ITMP2);
1531                         emit_ucomiss_reg_reg(cd, s1, s2);
1532                         M_CMOVB(REG_ITMP1, d);
1533                         M_CMOVA(REG_ITMP2, d);
1534                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1535                         emit_store_dst(jd, iptr, d);
1536                         break;
1537
1538                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1539                                           /* == => 0, < => 1, > => -1 */
1540
1541                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1542                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1543                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1544                         M_CLR(d);
1545                         M_MOV_IMM(1, REG_ITMP1);
1546                         M_MOV_IMM(-1, REG_ITMP2);
1547                         emit_ucomiss_reg_reg(cd, s1, s2);
1548                         M_CMOVB(REG_ITMP1, d);
1549                         M_CMOVA(REG_ITMP2, d);
1550                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1551                         emit_store_dst(jd, iptr, d);
1552                         break;
1553
1554                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1555                                           /* == => 0, < => 1, > => -1 */
1556
1557                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1558                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1559                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1560                         M_CLR(d);
1561                         M_MOV_IMM(1, REG_ITMP1);
1562                         M_MOV_IMM(-1, REG_ITMP2);
1563                         emit_ucomisd_reg_reg(cd, s1, s2);
1564                         M_CMOVB(REG_ITMP1, d);
1565                         M_CMOVA(REG_ITMP2, d);
1566                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1567                         emit_store_dst(jd, iptr, d);
1568                         break;
1569
1570                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1571                                           /* == => 0, < => 1, > => -1 */
1572
1573                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1574                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1575                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1576                         M_CLR(d);
1577                         M_MOV_IMM(1, REG_ITMP1);
1578                         M_MOV_IMM(-1, REG_ITMP2);
1579                         emit_ucomisd_reg_reg(cd, s1, s2);
1580                         M_CMOVB(REG_ITMP1, d);
1581                         M_CMOVA(REG_ITMP2, d);
1582                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1583                         emit_store_dst(jd, iptr, d);
1584                         break;
1585
1586
1587                 /* memory operations **************************************************/
1588
1589                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1590
1591                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1592                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1593                         gen_nullptr_check(s1);
1594                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1595                         emit_store_dst(jd, iptr, d);
1596                         break;
1597
1598                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1599
1600                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1601                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1602                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1603                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1604                                 gen_nullptr_check(s1);
1605                                 gen_bound_check;
1606                         }
1607                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1608                         emit_store_dst(jd, iptr, d);
1609                         break;
1610
1611                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1612
1613                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1614                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1615                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1616                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1617                                 gen_nullptr_check(s1);
1618                                 gen_bound_check;
1619                         }
1620                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1621                         emit_store_dst(jd, iptr, d);
1622                         break;
1623
1624                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1625
1626                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1627                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1628                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1629                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1630                                 gen_nullptr_check(s1);
1631                                 gen_bound_check;
1632                         }
1633                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1634                         emit_store_dst(jd, iptr, d);
1635                         break;
1636
1637                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1638
1639                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1640                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1641                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1642                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1643                                 gen_nullptr_check(s1);
1644                                 gen_bound_check;
1645                         }
1646                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1647                         emit_store_dst(jd, iptr, d);
1648                         break;
1649
1650                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1651
1652                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1653                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1654                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1655                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1656                                 gen_nullptr_check(s1);
1657                                 gen_bound_check;
1658                         }
1659                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1660                         emit_store_dst(jd, iptr, d);
1661                         break;
1662
1663                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1664
1665                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1666                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1667                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1668                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1669                                 gen_nullptr_check(s1);
1670                                 gen_bound_check;
1671                         }
1672                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1673                         emit_store_dst(jd, iptr, d);
1674                         break;
1675
1676                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1677
1678                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1679                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1680                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1681                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1682                                 gen_nullptr_check(s1);
1683                                 gen_bound_check;
1684                         }
1685                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1686                         emit_store_dst(jd, iptr, d);
1687                         break;
1688
1689                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1690
1691                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1692                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1693                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1694                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1695                                 gen_nullptr_check(s1);
1696                                 gen_bound_check;
1697                         }
1698                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1699                         emit_store_dst(jd, iptr, d);
1700                         break;
1701
1702
1703                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1704
1705                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1706                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1707                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1708                                 gen_nullptr_check(s1);
1709                                 gen_bound_check;
1710                         }
1711                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1712                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1713                         break;
1714
1715                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1716
1717                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1718                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1719                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1720                                 gen_nullptr_check(s1);
1721                                 gen_bound_check;
1722                         }
1723                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1724                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1725                         break;
1726
1727                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1728
1729                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1730                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1731                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1732                                 gen_nullptr_check(s1);
1733                                 gen_bound_check;
1734                         }
1735                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1736                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1737                         break;
1738
1739                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1740
1741                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1742                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1743                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1744                                 gen_nullptr_check(s1);
1745                                 gen_bound_check;
1746                         }
1747                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1748                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
1749                         break;
1750
1751                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1752
1753                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1754                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1755                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1756                                 gen_nullptr_check(s1);
1757                                 gen_bound_check;
1758                         }
1759                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1760                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
1761                         break;
1762
1763                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1764
1765                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1766                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1767                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1768                                 gen_nullptr_check(s1);
1769                                 gen_bound_check;
1770                         }
1771                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1772                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
1773                         break;
1774
1775                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1776
1777                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1778                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1779                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1780                                 gen_nullptr_check(s1);
1781                                 gen_bound_check;
1782                         }
1783                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1784                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
1785                         break;
1786
1787                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1788
1789                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1790                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1791                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1792                                 gen_nullptr_check(s1);
1793                                 gen_bound_check;
1794                         }
1795                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1796
1797                         M_MOV(s1, REG_A0);
1798                         M_MOV(s3, REG_A1);
1799                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
1800                         M_CALL(REG_ITMP1);
1801                         M_TEST(REG_RESULT);
1802                         M_BEQ(0);
1803                         codegen_add_arraystoreexception_ref(cd);
1804
1805                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1806                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1807                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1808                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1809                         break;
1810
1811
1812                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
1813
1814                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1815                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1816                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1817                                 gen_nullptr_check(s1);
1818                                 gen_bound_check;
1819                         }
1820                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1821                         break;
1822
1823                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
1824
1825                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1826                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1827                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1828                                 gen_nullptr_check(s1);
1829                                 gen_bound_check;
1830                         }
1831                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray, data[0]), s1, s2, 1);
1832                         break;
1833
1834                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
1835
1836                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1837                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1838                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1839                                 gen_nullptr_check(s1);
1840                                 gen_bound_check;
1841                         }
1842                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1843                         break;
1844
1845                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
1846
1847                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1848                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1849                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1850                                 gen_nullptr_check(s1);
1851                                 gen_bound_check;
1852                         }
1853                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray, data[0]), s1, s2, 2);
1854                         break;
1855
1856                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
1857
1858                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1859                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1860                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1861                                 gen_nullptr_check(s1);
1862                                 gen_bound_check;
1863                         }
1864
1865                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
1866                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1867                         } else {
1868                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
1869                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
1870                         }
1871                         break;
1872
1873                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
1874
1875                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1876                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1877                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1878                                 gen_nullptr_check(s1);
1879                                 gen_bound_check;
1880                         }
1881                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
1882                         break;
1883
1884
1885                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
1886
1887                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1888                                 uf        = iptr->sx.s23.s3.uf;
1889                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1890                                 disp      = dseg_add_unique_address(cd, NULL);
1891                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1892
1893                                 /* must be calculated before codegen_add_patch_ref */
1894
1895                                 if (opt_shownops)
1896                                         disp -= PATCHER_CALL_SIZE;
1897
1898 /*                              PROFILE_CYCLE_STOP; */
1899
1900                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1901
1902 /*                              PROFILE_CYCLE_START; */
1903                         }
1904                         else {
1905                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1906                                 fieldtype = fi->type;
1907                                 disp      = dseg_add_address(cd, &(fi->value));
1908                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1909
1910                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1911                                         PROFILE_CYCLE_STOP;
1912
1913                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1914
1915                                         if (opt_shownops)
1916                                                 disp -= PATCHER_CALL_SIZE;
1917
1918                                         PROFILE_CYCLE_START;
1919                                 }
1920                         }
1921
1922                         /* This approach is much faster than moving the field
1923                            address inline into a register. */
1924
1925                         M_ALD(REG_ITMP1, RIP, disp);
1926
1927                         switch (fieldtype) {
1928                         case TYPE_INT:
1929                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1930                                 M_ILD(d, REG_ITMP1, 0);
1931                                 break;
1932                         case TYPE_LNG:
1933                         case TYPE_ADR:
1934                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1935                                 M_LLD(d, REG_ITMP1, 0);
1936                                 break;
1937                         case TYPE_FLT:
1938                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1939                                 M_FLD(d, REG_ITMP1, 0);
1940                                 break;
1941                         case TYPE_DBL:
1942                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1943                                 M_DLD(d, REG_ITMP1, 0);
1944                                 break;
1945                         }
1946                         emit_store_dst(jd, iptr, d);
1947                         break;
1948
1949                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
1950
1951                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
1952                                 uf        = iptr->sx.s23.s3.uf;
1953                                 fieldtype = uf->fieldref->parseddesc.fd->type;
1954                                 disp      = dseg_add_unique_address(cd, NULL);
1955                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1956
1957                                 /* must be calculated before codegen_add_patch_ref */
1958
1959                                 if (opt_shownops)
1960                                         disp -= PATCHER_CALL_SIZE;
1961
1962 /*                              PROFILE_CYCLE_STOP; */
1963
1964                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
1965
1966 /*                              PROFILE_CYCLE_START; */
1967                         }
1968                         else {
1969                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
1970                                 fieldtype = fi->type;
1971                                 disp      = dseg_add_address(cd, &(fi->value));
1972                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
1973
1974                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
1975                                         PROFILE_CYCLE_STOP;
1976
1977                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
1978
1979                                         if (opt_shownops)
1980                                                 disp -= PATCHER_CALL_SIZE;
1981
1982                                         PROFILE_CYCLE_START;
1983                                 }
1984                         }
1985
1986                         /* This approach is much faster than moving the field
1987                            address inline into a register. */
1988
1989                         M_ALD(REG_ITMP1, RIP, disp);
1990
1991                         switch (fieldtype) {
1992                         case TYPE_INT:
1993                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1994                                 M_IST(s1, REG_ITMP1, 0);
1995                                 break;
1996                         case TYPE_LNG:
1997                         case TYPE_ADR:
1998                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
1999                                 M_LST(s1, REG_ITMP1, 0);
2000                                 break;
2001                         case TYPE_FLT:
2002                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
2003                                 M_FST(s1, REG_ITMP1, 0);
2004                                 break;
2005                         case TYPE_DBL:
2006                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
2007                                 M_DST(s1, REG_ITMP1, 0);
2008                                 break;
2009                         }
2010                         break;
2011
2012                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2013                                           /* val = value (in current instruction)     */
2014                                           /* following NOP)                           */
2015
2016                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2017                                 uf        = iptr->sx.s23.s3.uf;
2018                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2019                                 disp      = dseg_add_unique_address(cd, NULL);
2020                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2021
2022                                 /* must be calculated before codegen_add_patch_ref */
2023
2024                                 if (opt_shownops)
2025                                         disp -= PATCHER_CALL_SIZE;
2026
2027
2028 /*                              PROFILE_CYCLE_STOP; */
2029
2030                                 codegen_add_patch_ref(cd, PATCHER_get_putstatic, uf, disp);
2031
2032 /*                              PROFILE_CYCLE_START; */
2033                         }
2034                         else {
2035                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2036                                 fieldtype = fi->type;
2037                                 disp      = dseg_add_address(cd, &(fi->value));
2038                                 disp      = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2039
2040                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2041                                         PROFILE_CYCLE_STOP;
2042
2043                                         codegen_add_patch_ref(cd, PATCHER_clinit, fi->class, 0);
2044
2045                                         if (opt_shownops)
2046                                                 disp -= PATCHER_CALL_SIZE;
2047
2048                                         PROFILE_CYCLE_START;
2049                                 }
2050                         }
2051
2052                         /* This approach is much faster than moving the field
2053                            address inline into a register. */
2054
2055                         M_ALD(REG_ITMP1, RIP, disp);
2056
2057                         switch (fieldtype) {
2058                         case TYPE_INT:
2059                         case TYPE_FLT:
2060                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2061                                 break;
2062                         case TYPE_LNG:
2063                         case TYPE_ADR:
2064                         case TYPE_DBL:
2065                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
2066                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2067                                 else {
2068                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2069                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
2070                                 }
2071                                 break;
2072                         }
2073                         break;
2074
2075                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2076
2077                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2078                         gen_nullptr_check(s1);
2079
2080                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2081                                 uf        = iptr->sx.s23.s3.uf;
2082                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2083                                 disp      = 0;
2084
2085 /*                              PROFILE_CYCLE_STOP; */
2086
2087                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
2088
2089 /*                              PROFILE_CYCLE_START; */
2090                         }
2091                         else {
2092                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2093                                 fieldtype = fi->type;
2094                                 disp      = fi->offset;
2095                         }
2096
2097                         switch (fieldtype) {
2098                         case TYPE_INT:
2099                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
2100                                 M_ILD32(d, s1, disp);
2101                                 break;
2102                         case TYPE_LNG:
2103                         case TYPE_ADR:
2104                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
2105                                 M_LLD32(d, s1, disp);
2106                                 break;
2107                         case TYPE_FLT:
2108                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2109                                 M_FLD32(d, s1, disp);
2110                                 break;
2111                         case TYPE_DBL:
2112                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2113                                 M_DLD32(d, s1, disp);
2114                                 break;
2115                         }
2116                         emit_store_dst(jd, iptr, d);
2117                         break;
2118
2119                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2120
2121                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2122                         gen_nullptr_check(s1);
2123
2124                         s2 = emit_load_s2(jd, iptr, REG_IFTMP);
2125
2126                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2127                                 uf        = iptr->sx.s23.s3.uf;
2128                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2129                                 disp      = 0;
2130
2131 /*                              PROFILE_CYCLE_STOP; */
2132
2133                                 codegen_add_patch_ref(cd, PATCHER_get_putfield, uf, 0);
2134
2135 /*                              PROFILE_CYCLE_START; */
2136                         }
2137                         else {
2138                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2139                                 fieldtype = fi->type;
2140                                 disp      = fi->offset;
2141                         }
2142
2143                         switch (fieldtype) {
2144                         case TYPE_INT:
2145                                 M_IST32(s2, s1, disp);
2146                                 break;
2147                         case TYPE_LNG:
2148                         case TYPE_ADR:
2149                                 M_LST32(s2, s1, disp);
2150                                 break;
2151                         case TYPE_FLT:
2152                                 M_FST32(s2, s1, disp);
2153                                 break;
2154                         case TYPE_DBL:
2155                                 M_DST32(s2, s1, disp);
2156                                 break;
2157                         }
2158                         break;
2159
2160                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2161                                           /* val = value (in current instruction)     */
2162                                           /* following NOP)                           */
2163
2164                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2165                         gen_nullptr_check(s1);
2166
2167                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2168                                 uf        = iptr->sx.s23.s3.uf;
2169                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2170                                 disp      = 0;
2171
2172 /*                              PROFILE_CYCLE_STOP; */
2173
2174                                 codegen_add_patch_ref(cd, PATCHER_putfieldconst, uf, 0);
2175
2176 /*                              PROFILE_CYCLE_START; */
2177                         }
2178                         else {
2179                                 fi        = iptr->sx.s23.s3.fmiref->p.field;
2180                                 fieldtype = fi->type;
2181                                 disp      = fi->offset;
2182                         }
2183
2184                         switch (fieldtype) {
2185                         case TYPE_INT:
2186                         case TYPE_FLT:
2187                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2188                                 break;
2189                         case TYPE_LNG:
2190                         case TYPE_ADR:
2191                         case TYPE_DBL:
2192                                 /* XXX why no check for IS_IMM32? */
2193                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2194                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2195                                 break;
2196                         }
2197                         break;
2198
2199
2200                 /* branch operations **************************************************/
2201
2202                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2203
2204                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2205                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2206
2207                         PROFILE_CYCLE_STOP;
2208
2209 #ifdef ENABLE_VERIFIER
2210                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2211                                 uc = iptr->sx.s23.s2.uc;
2212
2213                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2214                         }
2215 #endif /* ENABLE_VERIFIER */
2216
2217                         M_CALL_IMM(0);                            /* passing exception pc */
2218                         M_POP(REG_ITMP2_XPC);
2219
2220                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2221                         M_JMP(REG_ITMP3);
2222                         break;
2223
2224                 case ICMD_GOTO:         /* ... ==> ...                                */
2225                 case ICMD_RET:          /* ... ==> ...                                */
2226
2227                         M_JMP_IMM(0);
2228                         codegen_add_branch_ref(cd, iptr->dst.block);
2229                         break;
2230
2231                 case ICMD_JSR:          /* ... ==> ...                                */
2232
2233                         M_JMP_IMM(0);
2234                         codegen_add_branch_ref(cd, iptr->sx.s23.s3.jsrtarget.block);
2235                         break;
2236
2237                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2238
2239                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2240                         M_TEST(s1);
2241                         M_BEQ(0);
2242                         codegen_add_branch_ref(cd, iptr->dst.block);
2243                         break;
2244
2245                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2246
2247                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2248                         M_TEST(s1);
2249                         M_BNE(0);
2250                         codegen_add_branch_ref(cd, iptr->dst.block);
2251                         break;
2252
2253                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2254
2255                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2256                         M_ICMP_IMM(iptr->sx.val.i, s1);
2257                         M_BEQ(0);
2258                         codegen_add_branch_ref(cd, iptr->dst.block);
2259                         break;
2260
2261                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2262
2263                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2264                         M_ICMP_IMM(iptr->sx.val.i, s1);
2265                         M_BLT(0);
2266                         codegen_add_branch_ref(cd, iptr->dst.block);
2267                         break;
2268
2269                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2270
2271                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2272                         M_ICMP_IMM(iptr->sx.val.i, s1);
2273                         M_BLE(0);
2274                         codegen_add_branch_ref(cd, iptr->dst.block);
2275                         break;
2276
2277                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2278
2279                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2280                         M_ICMP_IMM(iptr->sx.val.i, s1);
2281                         M_BNE(0);
2282                         codegen_add_branch_ref(cd, iptr->dst.block);
2283                         break;
2284
2285                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2286
2287                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2288                         M_ICMP_IMM(iptr->sx.val.i, s1);
2289                         M_BGT(0);
2290                         codegen_add_branch_ref(cd, iptr->dst.block);
2291                         break;
2292
2293                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2294
2295                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2296                         M_ICMP_IMM(iptr->sx.val.i, s1);
2297                         M_BGE(0);
2298                         codegen_add_branch_ref(cd, iptr->dst.block);
2299                         break;
2300
2301                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2302
2303                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2304                         if (IS_IMM32(iptr->sx.val.l))
2305                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2306                         else {
2307                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2308                                 M_LCMP(REG_ITMP2, s1);
2309                         }
2310                         M_BEQ(0);
2311                         codegen_add_branch_ref(cd, iptr->dst.block);
2312                         break;
2313
2314                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2315
2316                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2317                         if (IS_IMM32(iptr->sx.val.l))
2318                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2319                         else {
2320                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2321                                 M_LCMP(REG_ITMP2, s1);
2322                         }
2323                         M_BLT(0);
2324                         codegen_add_branch_ref(cd, iptr->dst.block);
2325                         break;
2326
2327                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2328
2329                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2330                         if (IS_IMM32(iptr->sx.val.l))
2331                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2332                         else {
2333                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2334                                 M_LCMP(REG_ITMP2, s1);
2335                         }
2336                         M_BLE(0);
2337                         codegen_add_branch_ref(cd, iptr->dst.block);
2338                         break;
2339
2340                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2341
2342                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2343                         if (IS_IMM32(iptr->sx.val.l))
2344                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2345                         else {
2346                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2347                                 M_LCMP(REG_ITMP2, s1);
2348                         }
2349                         M_BNE(0);
2350                         codegen_add_branch_ref(cd, iptr->dst.block);
2351                         break;
2352
2353                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2354
2355                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2356                         if (IS_IMM32(iptr->sx.val.l))
2357                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2358                         else {
2359                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2360                                 M_LCMP(REG_ITMP2, s1);
2361                         }
2362                         M_BGT(0);
2363                         codegen_add_branch_ref(cd, iptr->dst.block);
2364                         break;
2365
2366                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2367
2368                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2369                         if (IS_IMM32(iptr->sx.val.l))
2370                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2371                         else {
2372                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2373                                 M_LCMP(REG_ITMP2, s1);
2374                         }
2375                         M_BGE(0);
2376                         codegen_add_branch_ref(cd, iptr->dst.block);
2377                         break;
2378
2379                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2380
2381                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2382                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2383                         M_ICMP(s2, s1);
2384                         M_BEQ(0);
2385                         codegen_add_branch_ref(cd, iptr->dst.block);
2386                         break;
2387
2388                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2389                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2390
2391                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2392                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2393                         M_LCMP(s2, s1);
2394                         M_BEQ(0);
2395                         codegen_add_branch_ref(cd, iptr->dst.block);
2396                         break;
2397
2398                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2399
2400                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2401                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2402                         M_ICMP(s2, s1);
2403                         M_BNE(0);
2404                         codegen_add_branch_ref(cd, iptr->dst.block);
2405                         break;
2406
2407                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2408                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2409
2410                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2411                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2412                         M_LCMP(s2, s1);
2413                         M_BNE(0);
2414                         codegen_add_branch_ref(cd, iptr->dst.block);
2415                         break;
2416
2417                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2418
2419                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2420                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2421                         M_ICMP(s2, s1);
2422                         M_BLT(0);
2423                         codegen_add_branch_ref(cd, iptr->dst.block);
2424                         break;
2425
2426                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2427
2428                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2429                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2430                         M_LCMP(s2, s1);
2431                         M_BLT(0);
2432                         codegen_add_branch_ref(cd, iptr->dst.block);
2433                         break;
2434
2435                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2436
2437                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2438                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2439                         M_ICMP(s2, s1);
2440                         M_BGT(0);
2441                         codegen_add_branch_ref(cd, iptr->dst.block);
2442                         break;
2443
2444                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2445
2446                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2447                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2448                         M_LCMP(s2, s1);
2449                         M_BGT(0);
2450                         codegen_add_branch_ref(cd, iptr->dst.block);
2451                         break;
2452
2453                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2454
2455                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2456                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2457                         M_ICMP(s2, s1);
2458                         M_BLE(0);
2459                         codegen_add_branch_ref(cd, iptr->dst.block);
2460                         break;
2461
2462                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2463
2464                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2465                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2466                         M_LCMP(s2, s1);
2467                         M_BLE(0);
2468                         codegen_add_branch_ref(cd, iptr->dst.block);
2469                         break;
2470
2471                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2472
2473                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2474                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2475                         M_ICMP(s2, s1);
2476                         M_BGE(0);
2477                         codegen_add_branch_ref(cd, iptr->dst.block);
2478                         break;
2479
2480                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2481
2482                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2483                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2484                         M_LCMP(s2, s1);
2485                         M_BGE(0);
2486                         codegen_add_branch_ref(cd, iptr->dst.block);
2487                         break;
2488
2489                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2490                 case ICMD_LRETURN:
2491
2492                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2493                         M_INTMOVE(s1, REG_RESULT);
2494                         goto nowperformreturn;
2495
2496                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2497
2498                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2499                         M_INTMOVE(s1, REG_RESULT);
2500
2501 #ifdef ENABLE_VERIFIER
2502                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2503                                 uc = iptr->sx.s23.s2.uc;
2504
2505                                 PROFILE_CYCLE_STOP;
2506
2507                                 codegen_add_patch_ref(cd, PATCHER_athrow_areturn, uc, 0);
2508
2509                                 PROFILE_CYCLE_START;
2510                         }
2511 #endif /* ENABLE_VERIFIER */
2512                         goto nowperformreturn;
2513
2514                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2515                 case ICMD_DRETURN:
2516
2517                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2518                         M_FLTMOVE(s1, REG_FRESULT);
2519                         goto nowperformreturn;
2520
2521                 case ICMD_RETURN:      /* ...  ==> ...                                */
2522
2523 nowperformreturn:
2524                         {
2525                         s4 i, p;
2526
2527                         p = cd->stackframesize;
2528
2529 #if !defined(NDEBUG)
2530                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2531                                 emit_verbosecall_exit(jd);
2532 #endif /* !defined(NDEBUG) */
2533
2534 #if defined(ENABLE_THREADS)
2535                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2536                                 M_ALD(REG_A0, REG_SP, rd->memuse * 8);
2537
2538                                 /* we need to save the proper return value */
2539                                 switch (iptr->opc) {
2540                                 case ICMD_IRETURN:
2541                                 case ICMD_ARETURN:
2542                                 case ICMD_LRETURN:
2543                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2544                                         break;
2545                                 case ICMD_FRETURN:
2546                                 case ICMD_DRETURN:
2547                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2548                                         break;
2549                                 }
2550
2551                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2552                                 M_CALL(REG_ITMP1);
2553
2554                                 /* and now restore the proper return value */
2555                                 switch (iptr->opc) {
2556                                 case ICMD_IRETURN:
2557                                 case ICMD_ARETURN:
2558                                 case ICMD_LRETURN:
2559                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2560                                         break;
2561                                 case ICMD_FRETURN:
2562                                 case ICMD_DRETURN:
2563                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2564                                         break;
2565                                 }
2566                         }
2567 #endif
2568
2569                         /* restore saved registers */
2570
2571                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2572                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2573                         }
2574                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2575                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2576                         }
2577
2578                         /* deallocate stack */
2579
2580                         if (cd->stackframesize)
2581                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2582
2583                         /* generate method profiling code */
2584
2585                         PROFILE_CYCLE_STOP;
2586
2587                         M_RET;
2588                         }
2589                         break;
2590
2591
2592                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2593                         {
2594                                 s4 i, l;
2595                                 branch_target_t *table;
2596
2597                                 table = iptr->dst.table;
2598
2599                                 l = iptr->sx.s23.s2.tablelow;
2600                                 i = iptr->sx.s23.s3.tablehigh;
2601
2602                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2603                                 M_INTMOVE(s1, REG_ITMP1);
2604
2605                                 if (l != 0)
2606                                         M_ISUB_IMM(l, REG_ITMP1);
2607
2608                                 /* number of targets */
2609                                 i = i - l + 1;
2610
2611                 /* range check */
2612                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2613                                 M_BA(0);
2614
2615                                 codegen_add_branch_ref(cd, table[0].block); /* default target */
2616
2617                                 /* build jump table top down and use address of lowest entry */
2618
2619                                 table += i;
2620
2621                                 while (--i >= 0) {
2622                                         dseg_add_target(cd, table->block);
2623                                         --table;
2624                                 }
2625
2626                                 /* length of dataseg after last dseg_add_target is used
2627                                    by load */
2628
2629                                 M_MOV_IMM(0, REG_ITMP2);
2630                                 dseg_adddata(cd);
2631                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2632                                 M_JMP(REG_ITMP1);
2633                         }
2634                         break;
2635
2636
2637                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2638                         {
2639                                 s4 i;
2640                                 lookup_target_t *lookup;
2641
2642                                 lookup = iptr->dst.lookup;
2643
2644                                 i = iptr->sx.s23.s2.lookupcount;
2645
2646                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2647                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2648
2649                                 while (--i >= 0) {
2650                                         M_ICMP_IMM(lookup->value, s1);
2651                                         M_BEQ(0);
2652                                         codegen_add_branch_ref(cd, lookup->target.block);
2653                                         lookup++;
2654                                 }
2655
2656                                 M_JMP_IMM(0);
2657
2658                                 codegen_add_branch_ref(cd, iptr->sx.s23.s3.lookupdefault.block);
2659                         }
2660                         break;
2661
2662
2663                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2664
2665                         bte = iptr->sx.s23.s3.bte;
2666                         md  = bte->md;
2667                         goto gen_method;
2668
2669                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2670
2671                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2672                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2673                 case ICMD_INVOKEINTERFACE:
2674
2675                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2676                                 lm = NULL;
2677                                 um = iptr->sx.s23.s3.um;
2678                                 md = um->methodref->parseddesc.md;
2679                         }
2680                         else {
2681                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2682                                 um = NULL;
2683                                 md = lm->parseddesc;
2684                         }
2685
2686 gen_method:
2687                         s3 = md->paramcount;
2688
2689                         MCODECHECK((20 * s3) + 128);
2690
2691                         /* copy arguments to registers or stack location */
2692
2693                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2694                                 var = VAR(iptr->sx.s23.s2.args[s3]);
2695
2696                                 /* Already Preallocated (ARGVAR) ? */
2697                                 if (var->flags & PREALLOC)
2698                                         continue;
2699
2700                                 if (IS_INT_LNG_TYPE(var->type)) {
2701                                         if (!md->params[s3].inmemory) {
2702                                                 s1 = rd->argintregs[md->params[s3].regoff];
2703                                                 d = emit_load(jd, iptr, var, s1);
2704                                                 M_INTMOVE(d, s1);
2705                                         }
2706                                         else {
2707                                                 d = emit_load(jd, iptr, var, REG_ITMP1);
2708                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
2709                                         }
2710                                 }
2711                                 else {
2712                                         if (!md->params[s3].inmemory) {
2713                                                 s1 = rd->argfltregs[md->params[s3].regoff];
2714                                                 d = emit_load(jd, iptr, var, s1);
2715                                                 M_FLTMOVE(d, s1);
2716                                         }
2717                                         else {
2718                                                 d = emit_load(jd, iptr, var, REG_FTMP1);
2719
2720                                                 if (IS_2_WORD_TYPE(var->type))
2721                                                         M_DST(d, REG_SP, md->params[s3].regoff * 8);
2722                                                 else
2723                                                         M_FST(d, REG_SP, md->params[s3].regoff * 8);
2724                                         }
2725                                 }
2726                         }
2727
2728                         /* generate method profiling code */
2729
2730                         PROFILE_CYCLE_STOP;
2731
2732                         switch (iptr->opc) {
2733                         case ICMD_BUILTIN:
2734                                 M_MOV_IMM(bte->fp, REG_ITMP1);
2735                                 M_CALL(REG_ITMP1);
2736
2737                                 if (INSTRUCTION_MUST_CHECK(iptr)) {
2738                                         M_TEST(REG_RESULT);
2739                                         M_BEQ(0);
2740                                         codegen_add_fillinstacktrace_ref(cd);
2741                                 }
2742                                 break;
2743
2744                         case ICMD_INVOKESPECIAL:
2745                                 M_TEST(REG_A0);
2746                                 M_BEQ(0);
2747                                 codegen_add_nullpointerexception_ref(cd);
2748
2749                                 /* fall through */
2750
2751                         case ICMD_INVOKESTATIC:
2752                                 if (lm == NULL) {
2753                                         disp = dseg_add_unique_address(cd, NULL);
2754                                         disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2755
2756                                         /* must be calculated before codegen_add_patch_ref */
2757
2758                                         if (opt_shownops)
2759                                                 disp -= PATCHER_CALL_SIZE;
2760
2761                                         codegen_add_patch_ref(cd, PATCHER_invokestatic_special,
2762                                                                                   um, disp);
2763
2764 /*                                      a = 0; */
2765                                 }
2766                                 else {
2767                                         disp = dseg_add_functionptr(cd, lm->stubroutine);
2768                                         disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2769
2770 /*                                      a = (ptrint) lm->stubroutine; */
2771                                 }
2772
2773 /*                              M_MOV_IMM(a, REG_ITMP2); */
2774                                 M_ALD(REG_ITMP2, RIP, disp);
2775                                 M_CALL(REG_ITMP2);
2776                                 break;
2777
2778                         case ICMD_INVOKEVIRTUAL:
2779                                 gen_nullptr_check(REG_A0);
2780
2781                                 if (lm == NULL) {
2782                                         codegen_add_patch_ref(cd, PATCHER_invokevirtual, um, 0);
2783
2784                                         s1 = 0;
2785                                 }
2786                                 else
2787                                         s1 = OFFSET(vftbl_t, table[0]) +
2788                                                 sizeof(methodptr) * lm->vftblindex;
2789
2790                                 M_ALD(REG_METHODPTR, REG_A0,
2791                                           OFFSET(java_objectheader, vftbl));
2792                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
2793                                 M_CALL(REG_ITMP3);
2794                                 break;
2795
2796                         case ICMD_INVOKEINTERFACE:
2797                                 gen_nullptr_check(REG_A0);
2798
2799                                 if (lm == NULL) {
2800                                         codegen_add_patch_ref(cd, PATCHER_invokeinterface, um, 0);
2801
2802                                         s1 = 0;
2803                                         s2 = 0;
2804                                 }
2805                                 else {
2806                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
2807                                                 sizeof(methodptr) * lm->class->index;
2808
2809                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
2810                                 }
2811
2812                                 M_ALD(REG_METHODPTR, REG_A0,
2813                                           OFFSET(java_objectheader, vftbl));
2814                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
2815                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
2816                                 M_CALL(REG_ITMP3);
2817                                 break;
2818                         }
2819
2820                         /* generate method profiling code */
2821
2822                         PROFILE_CYCLE_START;
2823
2824                         /* store return value */
2825
2826                         d = md->returntype.type;
2827
2828                         if (d != TYPE_VOID) {
2829                                 if (IS_INT_LNG_TYPE(d)) {
2830                                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
2831                                         M_INTMOVE(REG_RESULT, s1);
2832                                 }
2833                                 else {
2834                                         s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
2835                                         M_FLTMOVE(REG_FRESULT, s1);
2836                                 }
2837                                 emit_store_dst(jd, iptr, s1);
2838                         }
2839                         break;
2840
2841
2842                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
2843
2844                                       /* val.a: (classinfo *) superclass              */
2845
2846                         /*  superclass is an interface:
2847                          *
2848                          *  OK if ((sub == NULL) ||
2849                          *         (sub->vftbl->interfacetablelength > super->index) &&
2850                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
2851                          *
2852                          *  superclass is a class:
2853                          *
2854                          *  OK if ((sub == NULL) || (0
2855                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
2856                          *         super->vftbl->diffval));
2857                          */
2858
2859                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
2860                                 /* object type cast-check */
2861
2862                                 classinfo *super;
2863                                 vftbl_t   *supervftbl;
2864                                 s4         superindex;
2865
2866                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2867                                         super = NULL;
2868                                         superindex = 0;
2869                                         supervftbl = NULL;
2870                                 }
2871                                 else {
2872                                         super = iptr->sx.s23.s3.c.cls;
2873                                         superindex = super->index;
2874                                         supervftbl = super->vftbl;
2875                                 }
2876
2877 #if defined(ENABLE_THREADS)
2878                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
2879 #endif
2880                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2881
2882                                 /* calculate interface checkcast code size */
2883
2884                                 s2 = 3; /* mov_membase_reg */
2885                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
2886
2887                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
2888                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
2889                                         3 /* test */ + 6 /* jcc */;
2890
2891                                 if (super == NULL)
2892                                         s2 += (opt_shownops ? 5 : 0);
2893
2894                                 /* calculate class checkcast code size */
2895
2896                                 s3 = 3; /* mov_membase_reg */
2897                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
2898                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
2899
2900 #if 0
2901                                 if (s1 != REG_ITMP1) {
2902                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
2903                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
2904                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
2905                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
2906                                         a += 3;    /* sub */
2907
2908                                 } else
2909 #endif
2910                                         {
2911                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
2912                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
2913                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
2914                                         }
2915
2916                                 s3 += 3 /* cmp */ + 6 /* jcc */;
2917
2918                                 if (super == NULL)
2919                                         s3 += (opt_shownops ? 5 : 0);
2920
2921                                 /* if class is not resolved, check which code to call */
2922
2923                                 if (super == NULL) {
2924                                         M_TEST(s1);
2925                                         M_BEQ(6 + (opt_shownops ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
2926
2927                                         codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
2928                                                                                   iptr->sx.s23.s3.c.ref, 0);
2929
2930                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
2931                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
2932                                         M_BEQ(s2 + 5);
2933                                 }
2934
2935                                 /* interface checkcast code */
2936
2937                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
2938                                         if (super != NULL) {
2939                                                 M_TEST(s1);
2940                                                 M_BEQ(s2);
2941                                         }
2942
2943                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2944
2945                                         if (super == NULL) {
2946                                                 codegen_add_patch_ref(cd,
2947                                                                                           PATCHER_checkcast_instanceof_interface,
2948                                                                                           iptr->sx.s23.s3.c.ref,
2949                                                                                           0);
2950                                         }
2951
2952                                         emit_movl_membase32_reg(cd, REG_ITMP2,
2953                                                                                           OFFSET(vftbl_t, interfacetablelength),
2954                                                                                           REG_ITMP3);
2955                                         /* XXX TWISTI: should this be int arithmetic? */
2956                                         M_LSUB_IMM32(superindex, REG_ITMP3);
2957                                         M_TEST(REG_ITMP3);
2958                                         M_BLE(0);
2959                                         codegen_add_classcastexception_ref(cd, s1);
2960                                         emit_mov_membase32_reg(cd, REG_ITMP2,
2961                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
2962                                                                                          superindex * sizeof(methodptr*),
2963                                                                                          REG_ITMP3);
2964                                         M_TEST(REG_ITMP3);
2965                                         M_BEQ(0);
2966                                         codegen_add_classcastexception_ref(cd, s1);
2967
2968                                         if (super == NULL)
2969                                                 M_JMP_IMM(s3);
2970                                 }
2971
2972                                 /* class checkcast code */
2973
2974                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
2975                                         if (super != NULL) {
2976                                                 M_TEST(s1);
2977                                                 M_BEQ(s3);
2978                                         }
2979
2980                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
2981
2982                                         if (super == NULL) {
2983                                                 codegen_add_patch_ref(cd, PATCHER_checkcast_class,
2984                                                                                           iptr->sx.s23.s3.c.ref,
2985                                                                                           0);
2986                                         }
2987
2988                                         M_MOV_IMM(supervftbl, REG_ITMP3);
2989 #if defined(ENABLE_THREADS)
2990                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
2991 #endif
2992                                         emit_movl_membase32_reg(cd, REG_ITMP2,
2993                                                                                           OFFSET(vftbl_t, baseval),
2994                                                                                           REG_ITMP2);
2995                                         /*                                      if (s1 != REG_ITMP1) { */
2996                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
2997                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
2998                                         /*                                                                                              REG_ITMP1); */
2999                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3000                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3001                                         /*                                                                                              REG_ITMP3); */
3002                                         /*  #if defined(ENABLE_THREADS) */
3003                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3004                                         /*  #endif */
3005                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
3006
3007                                         /*                                      } else { */
3008                                         emit_movl_membase32_reg(cd, REG_ITMP3,
3009                                                                                           OFFSET(vftbl_t, baseval),
3010                                                                                           REG_ITMP3);
3011                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3012                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3013                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3014                                         /*                                      } */
3015 #if defined(ENABLE_THREADS)
3016                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3017 #endif
3018                                         M_LCMP(REG_ITMP3, REG_ITMP2);
3019                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3020                                         codegen_add_classcastexception_ref(cd, s1);
3021                                 }
3022
3023                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
3024                         }
3025                         else {
3026                                 /* array type cast-check */
3027
3028                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
3029                                 M_INTMOVE(s1, REG_A0);
3030
3031                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3032                                         codegen_add_patch_ref(cd, PATCHER_builtin_arraycheckcast,
3033                                                                                   iptr->sx.s23.s3.c.ref, 0);
3034                                 }
3035
3036                                 M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
3037                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
3038                                 M_CALL(REG_ITMP1);
3039
3040                                 /* s1 may have been destroyed over the function call */
3041                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
3042                                 M_TEST(REG_RESULT);
3043                                 M_BEQ(0);
3044                                 codegen_add_classcastexception_ref(cd, s1);
3045
3046                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
3047                         }
3048
3049                         M_INTMOVE(s1, d);
3050                         emit_store_dst(jd, iptr, d);
3051                         break;
3052
3053                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3054
3055                                       /* val.a: (classinfo *) superclass              */
3056
3057                         /*  superclass is an interface:
3058                          *
3059                          *  return (sub != NULL) &&
3060                          *         (sub->vftbl->interfacetablelength > super->index) &&
3061                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3062                          *
3063                          *  superclass is a class:
3064                          *
3065                          *  return ((sub != NULL) && (0
3066                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3067                          *          super->vftbl->diffvall));
3068                          */
3069
3070                         {
3071                         classinfo *super;
3072                         vftbl_t   *supervftbl;
3073                         s4         superindex;
3074
3075                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3076                                 super = NULL;
3077                                 superindex = 0;
3078                                 supervftbl = NULL;
3079
3080                         } else {
3081                                 super = iptr->sx.s23.s3.c.cls;
3082                                 superindex = super->index;
3083                                 supervftbl = super->vftbl;
3084                         }
3085
3086 #if defined(ENABLE_THREADS)
3087             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3088 #endif
3089
3090                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
3091                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
3092                         if (s1 == d) {
3093                                 M_INTMOVE(s1, REG_ITMP1);
3094                                 s1 = REG_ITMP1;
3095                         }
3096
3097                         /* calculate interface instanceof code size */
3098
3099                         s2 = 3; /* mov_membase_reg */
3100                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3101                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3102                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3103                                 3 /* test */ + 4 /* setcc */;
3104
3105                         if (!super)
3106                                 s2 += (opt_shownops ? 5 : 0);
3107
3108                         /* calculate class instanceof code size */
3109
3110                         s3 = 3; /* mov_membase_reg */
3111                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3112                         s3 += 10; /* mov_imm_reg */
3113                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3114                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3115                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3116                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3117                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3118                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3119                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3120
3121                         if (super == NULL)
3122                                 s3 += (opt_shownops ? 5 : 0);
3123
3124                         emit_alu_reg_reg(cd, ALU_XOR, d, d);
3125
3126                         /* if class is not resolved, check which code to call */
3127
3128                         if (super == NULL) {
3129                                 emit_test_reg_reg(cd, s1, s1);
3130                                 emit_jcc(cd, CC_Z, (6 + (opt_shownops ? 5 : 0) +
3131                                                                                          7 + 6 + s2 + 5 + s3));
3132
3133                                 codegen_add_patch_ref(cd, PATCHER_checkcast_instanceof_flags,
3134                                                                           iptr->sx.s23.s3.c.ref, 0);
3135
3136                                 emit_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3137                                 emit_alul_imm_reg(cd, ALU_AND, ACC_INTERFACE, REG_ITMP3);
3138                                 emit_jcc(cd, CC_Z, s2 + 5);
3139                         }
3140
3141                         /* interface instanceof code */
3142
3143                         if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
3144                                 if (super != NULL) {
3145                                         emit_test_reg_reg(cd, s1, s1);
3146                                         emit_jcc(cd, CC_Z, s2);
3147                                 }
3148
3149                                 emit_mov_membase_reg(cd, s1,
3150                                                                          OFFSET(java_objectheader, vftbl),
3151                                                                          REG_ITMP1);
3152
3153                                 if (super == NULL) {
3154                                         codegen_add_patch_ref(cd,
3155                                                                                   PATCHER_checkcast_instanceof_interface,
3156                                                                                   iptr->sx.s23.s3.c.ref, 0);
3157                                 }
3158
3159                                 emit_movl_membase32_reg(cd, REG_ITMP1,
3160                                                                                   OFFSET(vftbl_t, interfacetablelength),
3161                                                                                   REG_ITMP3);
3162                                 emit_alu_imm32_reg(cd, ALU_SUB, superindex, REG_ITMP3);
3163                                 emit_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3164
3165                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3166
3167                                 emit_jcc(cd, CC_LE, a);
3168                                 emit_mov_membase32_reg(cd, REG_ITMP1,
3169                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3170                                                                                  superindex * sizeof(methodptr*),
3171                                                                                  REG_ITMP1);
3172                                 emit_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3173                                 emit_setcc_reg(cd, CC_NE, d);
3174
3175                                 if (!super)
3176                                         emit_jmp_imm(cd, s3);
3177                         }
3178
3179                         /* class instanceof code */
3180
3181                         if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
3182                                 if (super != NULL) {
3183                                         emit_test_reg_reg(cd, s1, s1);
3184                                         emit_jcc(cd, CC_E, s3);
3185                                 }
3186
3187                                 emit_mov_membase_reg(cd, s1,
3188                                                                            OFFSET(java_objectheader, vftbl),
3189                                                                            REG_ITMP1);
3190
3191                                 if (super == NULL) {
3192                                         codegen_add_patch_ref(cd, PATCHER_instanceof_class,
3193                                                                                   iptr->sx.s23.s3.c.ref, 0);
3194                                 }
3195
3196                                 emit_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3197 #if defined(ENABLE_THREADS)
3198                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3199 #endif
3200                                 emit_movl_membase_reg(cd, REG_ITMP1,
3201                                                                                 OFFSET(vftbl_t, baseval),
3202                                                                                 REG_ITMP1);
3203                                 emit_movl_membase_reg(cd, REG_ITMP2,
3204                                                                                 OFFSET(vftbl_t, diffval),
3205                                                                                 REG_ITMP3);
3206                                 emit_movl_membase_reg(cd, REG_ITMP2,
3207                                                                                 OFFSET(vftbl_t, baseval),
3208                                                                                 REG_ITMP2);
3209 #if defined(ENABLE_THREADS)
3210                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3211 #endif
3212                                 emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
3213                                 emit_alu_reg_reg(cd, ALU_XOR, d, d); /* may be REG_ITMP2 */
3214                                 emit_alu_reg_reg(cd, ALU_CMP, REG_ITMP3, REG_ITMP1);
3215                                 emit_setcc_reg(cd, CC_BE, d);
3216                         }
3217                         emit_store_dst(jd, iptr, d);
3218                         }
3219                         break;
3220
3221                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3222
3223                         /* check for negative sizes and copy sizes to stack if necessary  */
3224
3225                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
3226
3227                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
3228
3229                                 /* copy SAVEDVAR sizes to stack */
3230                                 var = VAR(iptr->sx.s23.s2.args[s1]);
3231
3232                                 /* Already Preallocated? */
3233                                 if (!(var->flags & PREALLOC)) {
3234                                         s2 = emit_load(jd, iptr, var, REG_ITMP1);
3235                                         M_LST(s2, REG_SP, s1 * 8);
3236                                 }
3237                         }
3238
3239                         /* is a patcher function set? */
3240
3241                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3242                                 codegen_add_patch_ref(cd, PATCHER_builtin_multianewarray,
3243                                                                           iptr->sx.s23.s3.c.ref, 0);
3244                         }
3245
3246                         /* a0 = dimension count */
3247
3248                         M_MOV_IMM(iptr->s1.argcount, REG_A0);
3249
3250                         /* a1 = classinfo */
3251
3252                         M_MOV_IMM(iptr->sx.s23.s3.c.cls, REG_A1);
3253
3254                         /* a2 = pointer to dimensions = stack pointer */
3255
3256                         M_MOV(REG_SP, REG_A2);
3257
3258                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3259                         M_CALL(REG_ITMP1);
3260
3261                         /* check for exception before result assignment */
3262
3263                         M_TEST(REG_RESULT);
3264                         M_BEQ(0);
3265                         codegen_add_fillinstacktrace_ref(cd);
3266
3267                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
3268                         M_INTMOVE(REG_RESULT, s1);
3269                         emit_store_dst(jd, iptr, s1);
3270                         break;
3271
3272                 default:
3273                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3274                         return false;
3275         } /* switch */
3276
3277         } /* for instruction */
3278
3279         MCODECHECK(512); /* XXX require a lower number? */
3280
3281         /* At the end of a basic block we may have to append some nops,
3282            because the patcher stub calling code might be longer than the
3283            actual instruction. So codepatching does not change the
3284            following block unintentionally. */
3285
3286         if (cd->mcodeptr < cd->lastmcodeptr) {
3287                 while (cd->mcodeptr < cd->lastmcodeptr) {
3288                         M_NOP;
3289                 }
3290         }
3291
3292         } /* if (bptr -> flags >= BBREACHED) */
3293         } /* for basic block */
3294
3295         dseg_createlinenumbertable(cd);
3296
3297         /* generate stubs */
3298
3299         emit_exception_stubs(jd);
3300         emit_patcher_stubs(jd);
3301 #if 0
3302         emit_replacement_stubs(jd);
3303 #endif
3304
3305         codegen_finish(jd);
3306
3307         /* everything's ok */
3308
3309         return true;
3310 }
3311
3312
3313 /* createcompilerstub **********************************************************
3314
3315    Creates a stub routine which calls the compiler.
3316
3317 *******************************************************************************/
3318
3319 #define COMPILERSTUB_DATASIZE    3 * SIZEOF_VOID_P
3320 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
3321
3322 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
3323
3324
3325 u1 *createcompilerstub(methodinfo *m)
3326 {
3327         u1          *s;                     /* memory to hold the stub            */
3328         ptrint      *d;
3329         codeinfo    *code;
3330         codegendata *cd;
3331         s4           dumpsize;
3332
3333         s = CNEW(u1, COMPILERSTUB_SIZE);
3334
3335         /* set data pointer and code pointer */
3336
3337         d = (ptrint *) s;
3338         s = s + COMPILERSTUB_DATASIZE;
3339
3340         /* mark start of dump memory area */
3341
3342         dumpsize = dump_size();
3343
3344         cd = DNEW(codegendata);
3345         cd->mcodeptr = s;
3346
3347         /* Store the codeinfo pointer in the same place as in the
3348            methodheader for compiled methods. */
3349
3350         code = code_codeinfo_new(m);
3351
3352         d[0] = (ptrint) asm_call_jit_compiler;
3353         d[1] = (ptrint) m;
3354         d[2] = (ptrint) code;
3355
3356         /* code for the stub */
3357
3358         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
3359         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
3360         M_JMP(REG_ITMP3);
3361
3362 #if defined(ENABLE_STATISTICS)
3363         if (opt_stat)
3364                 count_cstub_len += COMPILERSTUB_SIZE;
3365 #endif
3366
3367         /* release dump area */
3368
3369         dump_release(dumpsize);
3370
3371         return s;
3372 }
3373
3374
3375 /* createnativestub ************************************************************
3376
3377    Creates a stub routine which calls a native method.
3378
3379 *******************************************************************************/
3380
3381 u1 *createnativestub(functionptr f, jitdata *jd, methoddesc *nmd)
3382 {
3383         methodinfo   *m;
3384         codeinfo     *code;
3385         codegendata  *cd;
3386         registerdata *rd;
3387         methoddesc   *md;
3388         s4            nativeparams;
3389         s4            i, j;                 /* count variables                    */
3390         s4            t;
3391         s4            s1, s2;
3392
3393         /* get required compiler data */
3394
3395         m    = jd->m;
3396         code = jd->code;
3397         cd   = jd->cd;
3398         rd   = jd->rd;
3399
3400         /* initialize variables */
3401
3402         md = m->parseddesc;
3403         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
3404
3405         /* calculate stack frame size */
3406
3407         cd->stackframesize =
3408                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
3409                 sizeof(localref_table) / SIZEOF_VOID_P +
3410                 INT_ARG_CNT + FLT_ARG_CNT +
3411                 1 +                       /* functionptr, TODO: store in data segment */
3412                 nmd->memuse;
3413
3414         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
3415
3416         /* create method header */
3417
3418         (void) dseg_add_unique_address(cd, code);              /* CodeinfoPointer */
3419         (void) dseg_add_unique_s4(cd, cd->stackframesize * 8); /* FrameSize       */
3420         (void) dseg_add_unique_s4(cd, 0);                      /* IsSync          */
3421         (void) dseg_add_unique_s4(cd, 0);                      /* IsLeaf          */
3422         (void) dseg_add_unique_s4(cd, 0);                      /* IntSave         */
3423         (void) dseg_add_unique_s4(cd, 0);                      /* FltSave         */
3424         (void) dseg_addlinenumbertablesize(cd);
3425         (void) dseg_add_unique_s4(cd, 0);                      /* ExTableSize     */
3426
3427         /* generate native method profiling code */
3428
3429         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3430                 /* count frequency */
3431
3432                 M_MOV_IMM(code, REG_ITMP3);
3433                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3434         }
3435
3436         /* generate stub code */
3437
3438         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3439
3440 #if !defined(NDEBUG)
3441         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3442                 emit_verbosecall_enter(jd);
3443 #endif
3444
3445         /* get function address (this must happen before the stackframeinfo) */
3446
3447 #if !defined(WITH_STATIC_CLASSPATH)
3448         if (f == NULL)
3449                 codegen_add_patch_ref(cd, PATCHER_resolve_native, m, 0);
3450 #endif
3451
3452         M_MOV_IMM(f, REG_ITMP3);
3453
3454
3455         /* save integer and float argument registers */
3456
3457         for (i = 0, j = 0; i < md->paramcount; i++) {
3458                 if (!md->params[i].inmemory) {
3459                         s1 = md->params[i].regoff;
3460
3461                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3462                                 M_LST(rd->argintregs[s1], REG_SP, j * 8);
3463                         else
3464                                 M_DST(rd->argfltregs[s1], REG_SP, j * 8);
3465
3466                         j++;
3467                 }
3468         }
3469
3470         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3471
3472         /* create dynamic stack info */
3473
3474         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3475         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), REG_A1);
3476         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, REG_A2);
3477         M_ALD(REG_A3, REG_SP, cd->stackframesize * 8);
3478         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3479         M_CALL(REG_ITMP1);
3480
3481         /* restore integer and float argument registers */
3482
3483         for (i = 0, j = 0; i < md->paramcount; i++) {
3484                 if (!md->params[i].inmemory) {
3485                         s1 = md->params[i].regoff;
3486
3487                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3488                                 M_LLD(rd->argintregs[s1], REG_SP, j * 8);
3489                         else
3490                                 M_DLD(rd->argfltregs[s1], REG_SP, j * 8);
3491
3492                         j++;
3493                 }
3494         }
3495
3496         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3497
3498
3499         /* copy or spill arguments to new locations */
3500
3501         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3502                 t = md->paramtypes[i].type;
3503
3504                 if (IS_INT_LNG_TYPE(t)) {
3505                         if (!md->params[i].inmemory) {
3506                                 s1 = rd->argintregs[md->params[i].regoff];
3507
3508                                 if (!nmd->params[j].inmemory) {
3509                                         s2 = rd->argintregs[nmd->params[j].regoff];
3510                                         M_INTMOVE(s1, s2);
3511
3512                                 } else {
3513                                         s2 = nmd->params[j].regoff;
3514                                         M_LST(s1, REG_SP, s2 * 8);
3515                                 }
3516
3517                         } else {
3518                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3519                                 s2 = nmd->params[j].regoff;
3520                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
3521                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
3522                         }
3523
3524                 } else {
3525                         /* We only copy spilled float arguments, as the float argument    */
3526                         /* registers keep unchanged.                                      */
3527
3528                         if (md->params[i].inmemory) {
3529                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3530                                 s2 = nmd->params[j].regoff;
3531
3532                                 if (IS_2_WORD_TYPE(t)) {
3533                                         M_DLD(REG_FTMP1, REG_SP, s1 * 8);
3534                                         M_DST(REG_FTMP1, REG_SP, s2 * 8);
3535                                 } else {
3536                                         M_FLD(REG_FTMP1, REG_SP, s1 * 8);
3537                                         M_FST(REG_FTMP1, REG_SP, s2 * 8);
3538                                 }
3539                         }
3540                 }
3541         }
3542
3543         /* put class into second argument register */
3544
3545         if (m->flags & ACC_STATIC)
3546                 M_MOV_IMM(m->class, REG_A1);
3547
3548         /* put env into first argument register */
3549
3550         M_MOV_IMM(_Jv_env, REG_A0);
3551
3552         /* do the native function call */
3553
3554         M_CALL(REG_ITMP3);
3555
3556         /* save return value */
3557
3558         if (md->returntype.type != TYPE_VOID) {
3559                 if (IS_INT_LNG_TYPE(md->returntype.type))
3560                         M_LST(REG_RESULT, REG_SP, 0 * 8);
3561                 else
3562                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
3563         }
3564
3565 #if !defined(NDEBUG)
3566         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3567                 emit_verbosecall_exit(jd);
3568 #endif
3569
3570         /* remove native stackframe info */
3571
3572         M_ALEA(REG_SP, cd->stackframesize * 8, REG_A0);
3573         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3574         M_CALL(REG_ITMP1);
3575         M_MOV(REG_RESULT, REG_ITMP3);
3576
3577         /* restore return value */
3578
3579         if (md->returntype.type != TYPE_VOID) {
3580                 if (IS_INT_LNG_TYPE(md->returntype.type))
3581                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
3582                 else
3583                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3584         }
3585
3586         /* remove stackframe */
3587
3588         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3589
3590         /* test for exception */
3591
3592         M_TEST(REG_ITMP3);
3593         M_BNE(1);
3594         M_RET;
3595
3596         /* handle exception */
3597
3598         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3599         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3600         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3601
3602         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3603         M_JMP(REG_ITMP3);
3604
3605         /* generate patcher stubs */
3606
3607         emit_patcher_stubs(jd);
3608
3609         codegen_finish(jd);
3610
3611         return code->entrypoint;
3612 }
3613
3614
3615 /*
3616  * These are local overrides for various environment variables in Emacs.
3617  * Please do not remove this and leave it at the end of the file, where
3618  * Emacs will automagically detect them.
3619  * ---------------------------------------------------------------------
3620  * Local variables:
3621  * mode: c
3622  * indent-tabs-mode: t
3623  * c-basic-offset: 4
3624  * tab-width: 4
3625  * End:
3626  * vim:noexpandtab:sw=4:ts=4:
3627  */