src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31             Edwin Steiner
  32
  33    $Id: codegen.c 4943 2006-05-23 08:51:33Z twisti $
  34
  35 */
  36
  37
  38 #include "config.h"
  39
  40 #include <assert.h>
  41 #include <stdio.h>
  42
  43 #include "vm/types.h"
  44
  45 #include "md.h"
  46 #include "md-abi.h"
  47
  48 #include "vm/jit/x86_64/arch.h"
  49 #include "vm/jit/x86_64/codegen.h"
  50 #include "vm/jit/x86_64/md-emit.h"
  51
  52 #include "mm/memory.h"
  53 #include "native/jni.h"
  54 #include "native/native.h"
  55 #include "vm/builtin.h"
  56 #include "vm/exceptions.h"
  57 #include "vm/global.h"
  58 #include "vm/loader.h"
  59 #include "vm/options.h"
  60 #include "vm/statistics.h"
  61 #include "vm/stringlocal.h"
  62 #include "vm/vm.h"
  63 #include "vm/jit/asmpart.h"
  64 #include "vm/jit/codegen-common.h"
  65 #include "vm/jit/dseg.h"
  66 #include "vm/jit/emit.h"
  67 #include "vm/jit/jit.h"
  68 #include "vm/jit/methodheader.h"
  69 #include "vm/jit/parse.h"
  70 #include "vm/jit/patcher.h"
  71 #include "vm/jit/reg.h"
  72 #include "vm/jit/replace.h"
  73
  74 #if defined(ENABLE_LSRA)
  75 # include "vm/jit/allocator/lsra.h"
  76 #endif
  77
  78
  79 /* codegen *********************************************************************
  80
  81    Generates machine code.
  82
  83 *******************************************************************************/
  84
  85 bool codegen(jitdata *jd)
  86 {
  87         methodinfo         *m;
  88         codegendata        *cd;
  89         registerdata       *rd;
  90         s4                  len, s1, s2, s3, d, disp;
  91         u2                  currentline;
  92         ptrint              a;
  93         s4                  stackframesize;
  94         stackptr            src;
  95         varinfo            *var;
  96         basicblock         *bptr;
  97         instruction        *iptr;
  98         exceptiontable     *ex;
  99         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 100         builtintable_entry *bte;
 101         methoddesc         *md;
 102         rplpoint           *replacementpoint;
 103
 104         /* get required compiler data */
 105
 106         m  = jd->m;
 107         cd = jd->cd;
 108         rd = jd->rd;
 109
 110         /* prevent compiler warnings */
 111
 112         d = 0;
 113         lm = NULL;
 114         bte = NULL;
 115
 116         {
 117         s4 i, p, t, l;
 118         s4 savedregs_num;
 119
 120         savedregs_num = 0;
 121
 122         /* space to save used callee saved registers */
 123
 124         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 125         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 126
 127         stackframesize = rd->memuse + savedregs_num;
 128
 129 #if defined(ENABLE_THREADS)
 130         /* space to save argument of monitor_enter */
 131
 132         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 133                 stackframesize++;
 134 #endif
 135
 136     /* Keep stack of non-leaf functions 16-byte aligned for calls into native */
 137         /* code e.g. libc or jni (alignment problems with movaps).                */
 138
 139         if (!m->isleafmethod || opt_verbosecall)
 140                 stackframesize |= 0x1;
 141
 142         /* create method header */
 143
 144         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 145         (void) dseg_adds4(cd, stackframesize * 8);              /* FrameSize      */
 146
 147 #if defined(ENABLE_THREADS)
 148         /* IsSync contains the offset relative to the stack pointer for the
 149            argument of monitor_exit used in the exception handler. Since the
 150            offset could be zero and give a wrong meaning of the flag it is
 151            offset by one.
 152         */
 153
 154         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 155                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);        /* IsSync         */
 156         else
 157 #endif
 158                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 159
 160         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 161         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse);  /* IntSave        */
 162         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse);  /* FltSave        */
 163
 164         (void) dseg_addlinenumbertablesize(cd);
 165
 166         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 167
 168         /* create exception table */
 169
 170         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 171                 dseg_addtarget(cd, ex->start);
 172                 dseg_addtarget(cd, ex->end);
 173                 dseg_addtarget(cd, ex->handler);
 174                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 175         }
 176
 177         /* generate method profiling code */
 178
 179         if (opt_prof) {
 180                 /* count frequency */
 181
 182                 M_MOV_IMM(m, REG_ITMP3);
 183                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(methodinfo, frequency));
 184
 185                 PROFILE_CYCLE_START;
 186         }
 187
 188         /* create stack frame (if necessary) */
 189
 190         if (stackframesize)
 191                 M_ASUB_IMM(stackframesize * 8, REG_SP);
 192
 193         /* save used callee saved registers */
 194
 195         p = stackframesize;
 196         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 197                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 198         }
 199         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 200                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 201         }
 202
 203         /* take arguments out of register or stack frame */
 204
 205         md = m->parseddesc;
 206
 207         for (p = 0, l = 0; p < md->paramcount; p++) {
 208                 t = md->paramtypes[p].type;
 209                 var = &(rd->locals[l][t]);
 210                 l++;
 211                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 212                         l++;
 213                 if (var->type < 0)
 214                         continue;
 215                 s1 = md->params[p].regoff;
 216                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 217                         s2 = rd->argintregs[s1];
 218                         if (!md->params[p].inmemory) {           /* register arguments    */
 219                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 220                                         M_INTMOVE(s2, var->regoff);
 221
 222                                 } else {                             /* reg arg -> spilled    */
 223                                     M_LST(s2, REG_SP, var->regoff * 8);
 224                                 }
 225
 226                         } else {                                 /* stack arguments       */
 227                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 228                                         /* + 8 for return address */
 229                                         M_LLD(var->regoff, REG_SP, (stackframesize + s1) * 8 + 8);
 230
 231                                 } else {                             /* stack arg -> spilled  */
 232                                         var->regoff = stackframesize + s1 + 1;
 233                                 }
 234                         }
 235
 236                 } else {                                     /* floating args         */
 237                         if (!md->params[p].inmemory) {           /* register arguments    */
 238                                 s2 = rd->argfltregs[s1];
 239                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 240                                         M_FLTMOVE(s2, var->regoff);
 241
 242                                 } else {                                         /* reg arg -> spilled    */
 243                                         M_DST(s2, REG_SP, var->regoff * 8);
 244                                 }
 245
 246                         } else {                                 /* stack arguments       */
 247                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 248                                         M_DLD(var->regoff, REG_SP, (stackframesize + s1) * 8 + 8);
 249
 250                                 } else {
 251                                         var->regoff = stackframesize + s1 + 1;
 252                                 }
 253                         }
 254                 }
 255         }  /* end for */
 256
 257         /* save monitorenter argument */
 258
 259 #if defined(ENABLE_THREADS)
 260         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 261                 /* stack offset for monitor argument */
 262
 263                 s1 = rd->memuse;
 264
 265                 if (opt_verbosecall) {
 266                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 267
 268                         for (p = 0; p < INT_ARG_CNT; p++)
 269                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 270
 271                         for (p = 0; p < FLT_ARG_CNT; p++)
 272                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 273
 274                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 275                 }
 276
 277                 /* decide which monitor enter function to call */
 278
 279                 if (m->flags & ACC_STATIC) {
 280                         M_MOV_IMM(m->class, REG_ITMP1);
 281                         M_AST(REG_ITMP1, REG_SP, s1 * 8);
 282                         M_INTMOVE(REG_ITMP1, rd->argintregs[0]);
 283                         M_MOV_IMM(BUILTIN_staticmonitorenter, REG_ITMP1);
 284                         M_CALL(REG_ITMP1);
 285
 286                 } else {
 287                         M_TEST(rd->argintregs[0]);
 288                         M_BEQ(0);
 289                         codegen_add_nullpointerexception_ref(cd);
 290                         M_AST(rd->argintregs[0], REG_SP, s1 * 8);
 291                         M_MOV_IMM(BUILTIN_monitorenter, REG_ITMP1);
 292                         M_CALL(REG_ITMP1);
 293                 }
 294
 295                 if (opt_verbosecall) {
 296                         for (p = 0; p < INT_ARG_CNT; p++)
 297                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 298
 299                         for (p = 0; p < FLT_ARG_CNT; p++)
 300                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 301
 302                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 303                 }
 304         }
 305 #endif
 306
 307 #if !defined(NDEBUG)
 308         /* Copy argument registers to stack and call trace function with
 309            pointer to arguments on stack. */
 310
 311         if (opt_verbosecall) {
 312                 M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 313
 314                 /* save integer argument registers */
 315
 316                 for (p = 0; p < INT_ARG_CNT; p++)
 317                         M_LST(rd->argintregs[p], REG_SP, (1 + p) * 8);
 318
 319                 /* save float argument registers */
 320
 321                 for (p = 0; p < FLT_ARG_CNT; p++)
 322                         M_DST(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 323
 324                 /* save temporary registers for leaf methods */
 325
 326                 if (m->isleafmethod) {
 327                         for (p = 0; p < INT_TMP_CNT; p++)
 328                                 M_LST(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 329
 330                         for (p = 0; p < FLT_TMP_CNT; p++)
 331                                 M_DST(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 332                 }
 333
 334                 /* show integer hex code for float arguments */
 335
 336                 for (p = 0, l = 0; p < md->paramcount && p < INT_ARG_CNT; p++) {
 337                         /* If the paramtype is a float, we have to right shift all
 338                            following integer registers. */
 339
 340                         if (IS_FLT_DBL_TYPE(md->paramtypes[p].type)) {
 341                                 for (s1 = INT_ARG_CNT - 2; s1 >= p; s1--)
 342                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
 343
 344                                 emit_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 345                                 l++;
 346                         }
 347                 }
 348
 349                 M_MOV_IMM(m, REG_ITMP2);
 350                 M_AST(REG_ITMP2, REG_SP, 0 * 8);
 351                 M_MOV_IMM(builtin_trace_args, REG_ITMP1);
 352                 M_CALL(REG_ITMP1);
 353
 354                 /* restore integer argument registers */
 355
 356                 for (p = 0; p < INT_ARG_CNT; p++)
 357                         M_LLD(rd->argintregs[p], REG_SP, (1 + p) * 8);
 358
 359                 /* restore float argument registers */
 360
 361                 for (p = 0; p < FLT_ARG_CNT; p++)
 362                         M_DLD(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 363
 364                 /* restore temporary registers for leaf methods */
 365
 366                 if (m->isleafmethod) {
 367                         for (p = 0; p < INT_TMP_CNT; p++)
 368                                 M_LLD(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 369
 370                         for (p = 0; p < FLT_TMP_CNT; p++)
 371                                 M_DLD(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 372                 }
 373
 374                 M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 375         }
 376 #endif /* !defined(NDEBUG) */
 377
 378         }
 379
 380         /* end of header generation */
 381
 382         replacementpoint = jd->code->rplpoints;
 383
 384         /* walk through all basic blocks */
 385
 386         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 387
 388                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 389
 390                 if (bptr->flags >= BBREACHED) {
 391
 392                         /* branch resolving */
 393
 394                         branchref *bref;
 395                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 396                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 397                                                                   bref->branchpos,
 398                                                                   bptr->mpc);
 399                         }
 400
 401                 /* handle replacement points */
 402
 403                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 404                         replacementpoint->pc = (u1*)(ptrint)bptr->mpc; /* will be resolved later */
 405
 406                         replacementpoint++;
 407
 408                         assert(cd->lastmcodeptr <= cd->mcodeptr);
 409                         cd->lastmcodeptr = cd->mcodeptr + 5; /* 5 byte jmp patch */
 410                 }
 411
 412                 /* copy interface registers to their destination */
 413
 414                 src = bptr->instack;
 415                 len = bptr->indepth;
 416                 MCODECHECK(512);
 417
 418                 /* generate basicblock profiling code */
 419
 420                 if (opt_prof_bb) {
 421                         /* count frequency */
 422
 423                         M_MOV_IMM(m->bbfrequency, REG_ITMP2);
 424                         M_IINC_MEMBASE(REG_ITMP2, bptr->debug_nr * 4);
 425
 426                         /* if this is an exception handler, start profiling again */
 427
 428                         if (bptr->type == BBTYPE_EXH)
 429                                 PROFILE_CYCLE_START;
 430                 }
 431
 432 #if defined(ENABLE_LSRA)
 433                 if (opt_lsra) {
 434                         while (src != NULL) {
 435                                 len--;
 436                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 437                                         if (bptr->type == BBTYPE_SBR) {
 438 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 439                                                 if (!(src->flags & INMEMORY))
 440                                                         d = src->regoff;
 441                                                 else
 442                                                         d = REG_ITMP1;
 443                                                 M_POP(d);
 444                                                 emit_store(jd, NULL, src, d);
 445
 446                                         } else if (bptr->type == BBTYPE_EXH) {
 447 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 448                                                 if (!(src->flags & INMEMORY))
 449                                                         d= src->regoff;
 450                                                 else
 451                                                         d=REG_ITMP1;
 452                                                 M_INTMOVE(REG_ITMP1, d);
 453                                                 emit_store(jd, NULL, src, d);
 454                                         }
 455                                 }
 456                                 src = src->prev;
 457                         }
 458
 459                 } else {
 460 #endif
 461
 462                 while (src != NULL) {
 463                         len--;
 464                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 465                                 if (bptr->type == BBTYPE_SBR) {
 466                                         d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 467                                         M_POP(d);
 468                                         emit_store(jd, NULL, src, d);
 469
 470                                 } else if (bptr->type == BBTYPE_EXH) {
 471                                         d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 472                                         M_INTMOVE(REG_ITMP1, d);
 473                                         emit_store(jd, NULL, src, d);
 474                                 }
 475
 476                         } else {
 477                                 d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 478                                 if ((src->varkind != STACKVAR)) {
 479                                         s2 = src->type;
 480                                         if (IS_FLT_DBL_TYPE(s2)) {
 481                                                 s1 = rd->interfaces[len][s2].regoff;
 482
 483                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 484                                                         M_FLTMOVE(s1, d);
 485                                                 else
 486                                                         M_DLD(d, REG_SP, s1 * 8);
 487
 488                                                 emit_store(jd, NULL, src, d);
 489
 490                                         } else {
 491                                                 s1 = rd->interfaces[len][s2].regoff;
 492
 493                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 494                                                         M_INTMOVE(s1, d);
 495                                                 else
 496                                                         M_LLD(d, REG_SP, s1 * 8);
 497
 498                                                 emit_store(jd, NULL, src, d);
 499                                         }
 500                                 }
 501                         }
 502                         src = src->prev;
 503                 }
 504 #if defined(ENABLE_LSRA)
 505                 }
 506 #endif
 507                 /* walk through all instructions */
 508
 509                 src = bptr->instack;
 510                 len = bptr->icount;
 511                 currentline = 0;
 512
 513                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 514                         if (iptr->line != currentline) {
 515                                 dseg_addlinenumber(cd, iptr->line);
 516                                 currentline = iptr->line;
 517                         }
 518
 519                         MCODECHECK(1024);                         /* 1KB should be enough */
 520
 521                         switch (iptr->opc) {
 522                         case ICMD_INLINE_START: /* internal ICMDs                         */
 523                         case ICMD_INLINE_END:
 524                                 break;
 525
 526                         case ICMD_NOP:    /* ...  ==> ...                                 */
 527                                 break;
 528
 529                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 530
 531                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 532                                 M_TEST(s1);
 533                                 M_BEQ(0);
 534                                 codegen_add_nullpointerexception_ref(cd);
 535                                 break;
 536
 537                 /* constant operations ************************************************/
 538
 539                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 540                                       /* op1 = 0, val.i = constant                    */
 541
 542                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 543                         ICONST(d, iptr->val.i);
 544                         emit_store(jd, iptr, iptr->dst, d);
 545                         break;
 546
 547                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 548                                       /* op1 = 0, val.l = constant                    */
 549
 550                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 551                         LCONST(d, iptr->val.l);
 552                         emit_store(jd, iptr, iptr->dst, d);
 553                         break;
 554
 555                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 556                                       /* op1 = 0, val.f = constant                    */
 557
 558                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 559                         disp = dseg_addfloat(cd, iptr->val.f);
 560                         emit_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + disp, d);
 561                         emit_store(jd, iptr, iptr->dst, d);
 562                         break;
 563
 564                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 565                                       /* op1 = 0, val.d = constant                    */
 566
 567                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 568                         disp = dseg_adddouble(cd, iptr->val.d);
 569                         emit_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, d);
 570                         emit_store(jd, iptr, iptr->dst, d);
 571                         break;
 572
 573                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 574                                       /* op1 = 0, val.a = constant                    */
 575
 576                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 577
 578                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 579 /*                              PROFILE_CYCLE_STOP; */
 580
 581                                 codegen_addpatchref(cd, PATCHER_aconst,
 582                                                                         ICMD_ACONST_UNRESOLVED_CLASSREF(iptr), 0);
 583
 584                                 if (opt_showdisassemble) {
 585                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
 586                                 }
 587
 588 /*                              PROFILE_CYCLE_START; */
 589
 590                                 M_MOV_IMM(NULL, d);
 591
 592                         } else {
 593                                 if (iptr->val.a == 0)
 594                                         M_CLR(d);
 595                                 else
 596                                         M_MOV_IMM(iptr->val.a, d);
 597                         }
 598                         emit_store(jd, iptr, iptr->dst, d);
 599                         break;
 600
 601
 602                 /* load/store operations **********************************************/
 603
 604                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 605                                       /* op1 = local variable                         */
 606
 607                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 608                         if ((iptr->dst->varkind == LOCALVAR) &&
 609                                 (iptr->dst->varnum == iptr->op1))
 610                                 break;
 611                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 612                         if (var->flags & INMEMORY)
 613                                 M_ILD(d, REG_SP, var->regoff * 8);
 614                         else
 615                                 M_INTMOVE(var->regoff, d);
 616                         emit_store(jd, iptr, iptr->dst, d);
 617                         break;
 618
 619                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 620                 case ICMD_ALOAD:      /* op1 = local variable                         */
 621
 622                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 623                         if ((iptr->dst->varkind == LOCALVAR) &&
 624                                 (iptr->dst->varnum == iptr->op1))
 625                                 break;
 626                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 627                         if (var->flags & INMEMORY)
 628                                 M_LLD(d, REG_SP, var->regoff * 8);
 629                         else
 630                                 M_INTMOVE(var->regoff, d);
 631                         emit_store(jd, iptr, iptr->dst, d);
 632                         break;
 633
 634                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 635                                       /* op1 = local variable                         */
 636
 637                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 638                         if ((iptr->dst->varkind == LOCALVAR) &&
 639                             (iptr->dst->varnum == iptr->op1))
 640                                 break;
 641                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 642                         if (var->flags & INMEMORY)
 643                                 M_FLD(d, REG_SP, var->regoff * 8);
 644                         else
 645                                 M_FLTMOVE(var->regoff, d);
 646                         emit_store(jd, iptr, iptr->dst, d);
 647                         break;
 648
 649                 case ICMD_DLOAD:      /* ...  ==> ..., content of local variable      */
 650                                       /* op1 = local variable                         */
 651
 652                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 653                         if ((iptr->dst->varkind == LOCALVAR) &&
 654                             (iptr->dst->varnum == iptr->op1))
 655                                 break;
 656                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 657                         if (var->flags & INMEMORY)
 658                                 M_DLD(d, REG_SP, var->regoff * 8);
 659                         else
 660                                 M_FLTMOVE(var->regoff, d);
 661                         emit_store(jd, iptr, iptr->dst, d);
 662                         break;
 663
 664                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 665                                       /* op1 = local variable                         */
 666
 667                         if ((src->varkind == LOCALVAR) && (src->varnum == iptr->op1))
 668                                 break;
 669                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 670                         if (var->flags & INMEMORY) {
 671                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 672                                 M_IST(s1, REG_SP, var->regoff * 8);
 673                         } else {
 674                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 675                                 M_INTMOVE(s1, var->regoff);
 676                         }
 677                         break;
 678
 679                 case ICMD_LSTORE:     /* ..., value  ==> ...                          */
 680                 case ICMD_ASTORE:     /* op1 = local variable                         */
 681
 682                         if ((src->varkind == LOCALVAR) && (src->varnum == iptr->op1))
 683                                 break;
 684                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 685                         if (var->flags & INMEMORY) {
 686                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 687                                 M_LST(s1, REG_SP, var->regoff * 8);
 688                         } else {
 689                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 690                                 M_INTMOVE(s1, var->regoff);
 691                         }
 692                         break;
 693
 694                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 695                                       /* op1 = local variable                         */
 696
 697                         if ((src->varkind == LOCALVAR) &&
 698                             (src->varnum == iptr->op1)) {
 699                                 break;
 700                         }
 701                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 702                         if (var->flags & INMEMORY) {
 703                                 s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
 704                                 M_FST(s1, REG_SP, var->regoff * 8);
 705                         } else {
 706                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 707                                 M_FLTMOVE(s1, var->regoff);
 708                         }
 709                         break;
 710
 711                 case ICMD_DSTORE:     /* ..., value  ==> ...                          */
 712                                       /* op1 = local variable                         */
 713
 714                         if ((src->varkind == LOCALVAR) &&
 715                             (src->varnum == iptr->op1)) {
 716                                 break;
 717                         }
 718                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 719                         if (var->flags & INMEMORY) {
 720                                 s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
 721                                 M_DST(s1, REG_SP, var->regoff * 8);
 722                         } else {
 723                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 724                                 M_FLTMOVE(s1, var->regoff);
 725                         }
 726                         break;
 727
 728
 729                 /* pop/dup/swap operations ********************************************/
 730
 731                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 732
 733                 case ICMD_POP:        /* ..., value  ==> ...                          */
 734                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 735                         break;
 736
 737                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 738                         M_COPY(src, iptr->dst);
 739                         break;
 740
 741                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 742
 743                         M_COPY(src,       iptr->dst);
 744                         M_COPY(src->prev, iptr->dst->prev);
 745                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 746                         break;
 747
 748                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 749
 750                         M_COPY(src,             iptr->dst);
 751                         M_COPY(src->prev,       iptr->dst->prev);
 752                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 753                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 754                         break;
 755
 756                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 757
 758                         M_COPY(src,       iptr->dst);
 759                         M_COPY(src->prev, iptr->dst->prev);
 760                         break;
 761
 762                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 763
 764                         M_COPY(src,             iptr->dst);
 765                         M_COPY(src->prev,       iptr->dst->prev);
 766                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 767                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 768                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 769                         break;
 770
 771                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 772
 773                         M_COPY(src,                   iptr->dst);
 774                         M_COPY(src->prev,             iptr->dst->prev);
 775                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 776                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 777                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 778                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 779                         break;
 780
 781                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 782
 783                         M_COPY(src,       iptr->dst->prev);
 784                         M_COPY(src->prev, iptr->dst);
 785                         break;
 786
 787
 788                 /* integer operations *************************************************/
 789
 790                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 791
 792                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 793                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 794                         M_INTMOVE(s1, d);
 795                         M_INEG(d);
 796                         emit_store(jd, iptr, iptr->dst, d);
 797                         break;
 798
 799                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 800
 801                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 802                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 803                         M_INTMOVE(s1, d);
 804                         M_LNEG(d);
 805                         emit_store(jd, iptr, iptr->dst, d);
 806                         break;
 807
 808                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 809
 810                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 811                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 812                         M_ISEXT(s1, d);
 813                         emit_store(jd, iptr, iptr->dst, d);
 814                         break;
 815
 816                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 817
 818                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 819                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 820                         M_IMOV(s1, d);
 821                         emit_store(jd, iptr, iptr->dst, d);
 822                         break;
 823
 824                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 825
 826                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 827                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 828                         M_BSEXT(s1, d);
 829                         emit_store(jd, iptr, iptr->dst, d);
 830                         break;
 831
 832                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 833
 834                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 835                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 836                         M_CZEXT(s1, d);
 837                         emit_store(jd, iptr, iptr->dst, d);
 838                         break;
 839
 840                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 841
 842                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 843                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 844                         M_SSEXT(s1, d);
 845                         emit_store(jd, iptr, iptr->dst, d);
 846                         break;
 847
 848
 849                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 850
 851                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 852                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 853                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 854                         if (s2 == d)
 855                                 M_IADD(s1, d);
 856                         else {
 857                                 M_INTMOVE(s1, d);
 858                                 M_IADD(s2, d);
 859                         }
 860                         emit_store(jd, iptr, iptr->dst, d);
 861                         break;
 862
 863                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 864                                       /* val.i = constant                             */
 865
 866                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 867                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 868                         M_INTMOVE(s1, d);
 869                         M_IADD_IMM(iptr->val.i, d);
 870                         emit_store(jd, iptr, iptr->dst, d);
 871                         break;
 872
 873                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 874
 875                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 876                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 877                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 878                         if (s2 == d)
 879                                 M_LADD(s1, d);
 880                         else {
 881                                 M_INTMOVE(s1, d);
 882                                 M_LADD(s2, d);
 883                         }
 884                         emit_store(jd, iptr, iptr->dst, d);
 885                         break;
 886
 887                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 888                                       /* val.l = constant                             */
 889
 890                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 891                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 892                         M_INTMOVE(s1, d);
 893                         if (IS_IMM32(iptr->val.l))
 894                                 M_LADD_IMM(iptr->val.l, d);
 895                         else {
 896                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
 897                                 M_LADD(REG_ITMP2, d);
 898                         }
 899                         emit_store(jd, iptr, iptr->dst, d);
 900                         break;
 901
 902                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 903
 904                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 905                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 906                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 907                         if (s2 == d) {
 908                                 M_INTMOVE(s1, REG_ITMP1);
 909                                 M_ISUB(s2, REG_ITMP1);
 910                                 M_INTMOVE(REG_ITMP1, d);
 911                         } else {
 912                                 M_INTMOVE(s1, d);
 913                                 M_ISUB(s2, d);
 914                         }
 915                         emit_store(jd, iptr, iptr->dst, d);
 916                         break;
 917
 918                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 919                                       /* val.i = constant                             */
 920
 921                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 922                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 923                         M_INTMOVE(s1, d);
 924                         M_ISUB_IMM(iptr->val.i, d);
 925                         emit_store(jd, iptr, iptr->dst, d);
 926                         break;
 927
 928                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 929
 930                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 931                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 932                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 933                         if (s2 == d) {
 934                                 M_INTMOVE(s1, REG_ITMP1);
 935                                 M_LSUB(s2, REG_ITMP1);
 936                                 M_INTMOVE(REG_ITMP1, d);
 937                         } else {
 938                                 M_INTMOVE(s1, d);
 939                                 M_LSUB(s2, d);
 940                         }
 941                         emit_store(jd, iptr, iptr->dst, d);
 942                         break;
 943
 944                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 945                                       /* val.l = constant                             */
 946
 947                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 948                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 949                         M_INTMOVE(s1, d);
 950                         if (IS_IMM32(iptr->val.l))
 951                                 M_LSUB_IMM(iptr->val.l, d);
 952                         else {
 953                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
 954                                 M_LSUB(REG_ITMP2, d);
 955                         }
 956                         emit_store(jd, iptr, iptr->dst, d);
 957                         break;
 958
 959                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 960
 961                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 962                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 963                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 964                         if (s2 == d)
 965                                 M_IMUL(s1, d);
 966                         else {
 967                                 M_INTMOVE(s1, d);
 968                                 M_IMUL(s2, d);
 969                         }
 970                         emit_store(jd, iptr, iptr->dst, d);
 971                         break;
 972
 973                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 974                                       /* val.i = constant                             */
 975
 976                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 977                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 978                         if (iptr->val.i == 2) {
 979                                 M_INTMOVE(s1, d);
 980                                 M_ISLL_IMM(1, d);
 981                         } else
 982                                 M_IMUL_IMM(s1, iptr->val.i, d);
 983                         emit_store(jd, iptr, iptr->dst, d);
 984                         break;
 985
 986                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 987
 988                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 989                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 990                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 991                         if (s2 == d)
 992                                 M_LMUL(s1, d);
 993                         else {
 994                                 M_INTMOVE(s1, d);
 995                                 M_LMUL(s2, d);
 996                         }
 997                         emit_store(jd, iptr, iptr->dst, d);
 998                         break;
 999
1000                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1001                                       /* val.l = constant                             */
1002
1003                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1004                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1005                         if (IS_IMM32(iptr->val.l))
1006                                 M_LMUL_IMM(s1, iptr->val.l, d);
1007                         else {
1008                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
1009                                 M_INTMOVE(s1, d);
1010                                 M_LMUL(REG_ITMP2, d);
1011                         }
1012                         emit_store(jd, iptr, iptr->dst, d);
1013                         break;
1014
1015                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1016
1017                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1018                 if (src->prev->flags & INMEMORY)
1019                                 M_ILD(RAX, REG_SP, src->prev->regoff * 8);
1020                         else
1021                                 M_INTMOVE(src->prev->regoff, RAX);
1022
1023                         if (src->flags & INMEMORY)
1024                                 M_ILD(REG_ITMP3, REG_SP, src->regoff * 8);
1025                         else
1026                                 M_INTMOVE(src->regoff, REG_ITMP3);
1027
1028                         if (checknull) {
1029                                 M_ITEST(REG_ITMP3);
1030                                 M_BEQ(0);
1031                                 codegen_add_arithmeticexception_ref(cd);
1032                         }
1033
1034                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1035                         emit_jcc(cd, CC_NE, 4 + 6);
1036                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
1037                         emit_jcc(cd, CC_E, 3 + 1 + 3);                /* 6 bytes */
1038
1039                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1040                         emit_cltd(cd);
1041                         emit_idivl_reg(cd, REG_ITMP3);
1042
1043                         if (iptr->dst->flags & INMEMORY) {
1044                                 emit_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1045                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1046
1047                         } else {
1048                                 M_INTMOVE(RAX, iptr->dst->regoff);
1049
1050                                 if (iptr->dst->regoff != RDX) {
1051                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1052                                 }
1053                         }
1054                         break;
1055
1056                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1057
1058                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1059                         if (src->prev->flags & INMEMORY)
1060                                 M_ILD(RAX, REG_SP, src->prev->regoff * 8);
1061                         else
1062                                 M_INTMOVE(src->prev->regoff, RAX);
1063
1064                         if (src->flags & INMEMORY)
1065                                 M_ILD(REG_ITMP3, REG_SP, src->regoff * 8);
1066                         else
1067                                 M_INTMOVE(src->regoff, REG_ITMP3);
1068
1069                         if (checknull) {
1070                                 M_ITEST(REG_ITMP3);
1071                                 M_BEQ(0);
1072                                 codegen_add_arithmeticexception_ref(cd);
1073                         }
1074
1075                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1076
1077                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1078                         emit_jcc(cd, CC_NE, 2 + 4 + 6);
1079
1080
1081                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
1082                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
1083                         emit_jcc(cd, CC_E, 1 + 3);                    /* 6 bytes */
1084
1085                         emit_cltd(cd);
1086                         emit_idivl_reg(cd, REG_ITMP3);
1087
1088                         if (iptr->dst->flags & INMEMORY) {
1089                                 emit_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1090                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1091
1092                         } else {
1093                                 M_INTMOVE(RDX, iptr->dst->regoff);
1094
1095                                 if (iptr->dst->regoff != RDX) {
1096                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1097                                 }
1098                         }
1099                         break;
1100
1101                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1102                                       /* val.i = constant                             */
1103
1104                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1105                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1106                         M_INTMOVE(s1, REG_ITMP1);
1107                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1108                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1109                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
1110                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->val.i, REG_ITMP1);
1111                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1112                         emit_store(jd, iptr, iptr->dst, d);
1113                         break;
1114
1115                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1116                                       /* val.i = constant                             */
1117
1118                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1119                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1120                         M_INTMOVE(s1, REG_ITMP1);
1121                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1122                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1123                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
1124                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->val.i), REG_ITMP2);
1125                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
1126                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1127                         emit_store(jd, iptr, iptr->dst, d);
1128                         break;
1129
1130
1131                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1132
1133                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1134
1135                 if (src->prev->flags & INMEMORY)
1136                                 M_LLD(RAX, REG_SP, src->prev->regoff * 8);
1137                         else
1138                                 M_INTMOVE(src->prev->regoff, RAX);
1139
1140                         if (src->flags & INMEMORY)
1141                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1142                         else
1143                                 M_INTMOVE(src->regoff, REG_ITMP3);
1144
1145                         if (checknull) {
1146                                 M_TEST(REG_ITMP3);
1147                                 M_BEQ(0);
1148                                 codegen_add_arithmeticexception_ref(cd);
1149                         }
1150
1151                         /* check as described in jvm spec */
1152                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1153                         M_LCMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, RAX);
1154                         M_BNE(4 + 6);
1155                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1156                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
1157
1158                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1159                         emit_cqto(cd);
1160                         emit_idiv_reg(cd, REG_ITMP3);
1161
1162                         if (iptr->dst->flags & INMEMORY) {
1163                                 M_LST(RAX, REG_SP, iptr->dst->regoff * 8);
1164                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1165
1166                         } else {
1167                                 M_INTMOVE(RAX, iptr->dst->regoff);
1168
1169                                 if (iptr->dst->regoff != RDX) {
1170                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1171                                 }
1172                         }
1173                         break;
1174
1175                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1176
1177                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1178                         if (src->prev->flags & INMEMORY)
1179                                 M_LLD(REG_ITMP1, REG_SP, src->prev->regoff * 8);
1180                         else
1181                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1182
1183                         if (src->flags & INMEMORY)
1184                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1185                         else
1186                                 M_INTMOVE(src->regoff, REG_ITMP3);
1187
1188                         if (checknull) {
1189                                 M_ITEST(REG_ITMP3);
1190                                 M_BEQ(0);
1191                                 codegen_add_arithmeticexception_ref(cd);
1192                         }
1193
1194                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1195
1196                         /* check as described in jvm spec */
1197                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1198                         M_LCMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP1);
1199                         M_BNE(3 + 4 + 6);
1200
1201 #if 0
1202                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
1203 #endif
1204                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
1205                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1206                         M_BEQ(2 + 3);                                          /* 6 bytes */
1207
1208                         emit_cqto(cd);
1209                         emit_idiv_reg(cd, REG_ITMP3);
1210
1211                         if (iptr->dst->flags & INMEMORY) {
1212                                 M_LST(RDX, REG_SP, iptr->dst->regoff * 8);
1213                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1214
1215                         } else {
1216                                 M_INTMOVE(RDX, iptr->dst->regoff);
1217
1218                                 if (iptr->dst->regoff != RDX) {
1219                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1220                                 }
1221                         }
1222                         break;
1223
1224                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1225                                       /* val.i = constant                             */
1226
1227                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1228                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1229                         M_INTMOVE(s1, REG_ITMP1);
1230                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1231                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1232                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
1233                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->val.i, REG_ITMP1);
1234                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1235                         emit_store(jd, iptr, iptr->dst, d);
1236                         break;
1237
1238                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1239                                       /* val.l = constant                             */
1240
1241                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1242                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1243                         M_INTMOVE(s1, REG_ITMP1);
1244                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1245                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1246                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
1247                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->val.i), REG_ITMP2);
1248                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
1249                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1250                         emit_store(jd, iptr, iptr->dst, d);
1251                         break;
1252
1253                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1254
1255                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1256                         emit_ishift(cd, SHIFT_SHL, src, iptr);
1257                         break;
1258
1259                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1260                                       /* val.i = constant                             */
1261
1262                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1263                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1264                         M_INTMOVE(s1, d);
1265                         M_ISLL_IMM(iptr->val.i, d);
1266                         emit_store(jd, iptr, iptr->dst, d);
1267                         break;
1268
1269                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1270
1271                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1272                         emit_ishift(cd, SHIFT_SAR, src, iptr);
1273                         break;
1274
1275                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1276                                       /* val.i = constant                             */
1277
1278                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1279                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1280                         M_INTMOVE(s1, d);
1281                         M_ISRA_IMM(iptr->val.i, d);
1282                         emit_store(jd, iptr, iptr->dst, d);
1283                         break;
1284
1285                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1286
1287                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1288                         emit_ishift(cd, SHIFT_SHR, src, iptr);
1289                         break;
1290
1291                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1292                                       /* val.i = constant                             */
1293
1294                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1295                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1296                         M_INTMOVE(s1, d);
1297                         M_ISRL_IMM(iptr->val.i, d);
1298                         emit_store(jd, iptr, iptr->dst, d);
1299                         break;
1300
1301                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1302
1303                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1304                         emit_lshift(cd, SHIFT_SHL, src, iptr);
1305                         break;
1306
1307         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1308                                           /* val.i = constant                             */
1309
1310                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1311                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1312                         M_INTMOVE(s1, d);
1313                         M_LSLL_IMM(iptr->val.i, d);
1314                         emit_store(jd, iptr, iptr->dst, d);
1315                         break;
1316
1317                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1318
1319                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1320                         emit_lshift(cd, SHIFT_SAR, src, iptr);
1321                         break;
1322
1323                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1324                                       /* val.i = constant                             */
1325
1326                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1327                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1328                         M_INTMOVE(s1, d);
1329                         M_LSRA_IMM(iptr->val.i, d);
1330                         emit_store(jd, iptr, iptr->dst, d);
1331                         break;
1332
1333                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1334
1335                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1336                         emit_lshift(cd, SHIFT_SHR, src, iptr);
1337                         break;
1338
1339                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1340                                       /* val.l = constant                             */
1341
1342                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1343                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1344                         M_INTMOVE(s1, d);
1345                         M_LSRL_IMM(iptr->val.i, d);
1346                         emit_store(jd, iptr, iptr->dst, d);
1347                         break;
1348
1349                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1350
1351                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1352                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1353                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1354                         if (s2 == d)
1355                                 M_IAND(s1, d);
1356                         else {
1357                                 M_INTMOVE(s1, d);
1358                                 M_IAND(s2, d);
1359                         }
1360                         emit_store(jd, iptr, iptr->dst, d);
1361                         break;
1362
1363                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1364                                       /* val.i = constant                             */
1365
1366                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1367                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1368                         M_INTMOVE(s1, d);
1369                         M_IAND_IMM(iptr->val.i, d);
1370                         emit_store(jd, iptr, iptr->dst, d);
1371                         break;
1372
1373                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1374
1375                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1376                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1377                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1378                         if (s2 == d)
1379                                 M_LAND(s1, d);
1380                         else {
1381                                 M_INTMOVE(s1, d);
1382                                 M_LAND(s2, d);
1383                         }
1384                         emit_store(jd, iptr, iptr->dst, d);
1385                         break;
1386
1387                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1388                                       /* val.l = constant                             */
1389
1390                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1391                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1392                         M_INTMOVE(s1, d);
1393                         if (IS_IMM32(iptr->val.l))
1394                                 M_LAND_IMM(iptr->val.l, d);
1395                         else {
1396                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
1397                                 M_LAND(REG_ITMP2, d);
1398                         }
1399                         emit_store(jd, iptr, iptr->dst, d);
1400                         break;
1401
1402                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1403
1404                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1405                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1406                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1407                         if (s2 == d)
1408                                 M_IOR(s1, d);
1409                         else {
1410                                 M_INTMOVE(s1, d);
1411                                 M_IOR(s2, d);
1412                         }
1413                         emit_store(jd, iptr, iptr->dst, d);
1414                         break;
1415
1416                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1417                                       /* val.i = constant                             */
1418
1419                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1420                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1421                         M_INTMOVE(s1, d);
1422                         M_IOR_IMM(iptr->val.i, d);
1423                         emit_store(jd, iptr, iptr->dst, d);
1424                         break;
1425
1426                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1427
1428                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1429                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1430                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1431                         if (s2 == d)
1432                                 M_LOR(s1, d);
1433                         else {
1434                                 M_INTMOVE(s1, d);
1435                                 M_LOR(s2, d);
1436                         }
1437                         emit_store(jd, iptr, iptr->dst, d);
1438                         break;
1439
1440                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1441                                       /* val.l = constant                             */
1442
1443                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1444                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1445                         M_INTMOVE(s1, d);
1446                         if (IS_IMM32(iptr->val.l))
1447                                 M_LOR_IMM(iptr->val.l, d);
1448                         else {
1449                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
1450                                 M_LOR(REG_ITMP2, d);
1451                         }
1452                         emit_store(jd, iptr, iptr->dst, d);
1453                         break;
1454
1455                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1456
1457                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1458                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1459                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1460                         if (s2 == d)
1461                                 M_IXOR(s1, d);
1462                         else {
1463                                 M_INTMOVE(s1, d);
1464                                 M_IXOR(s2, d);
1465                         }
1466                         emit_store(jd, iptr, iptr->dst, d);
1467                         break;
1468
1469                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1470                                       /* val.i = constant                             */
1471
1472                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1473                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1474                         M_INTMOVE(s1, d);
1475                         M_IXOR_IMM(iptr->val.i, d);
1476                         emit_store(jd, iptr, iptr->dst, d);
1477                         break;
1478
1479                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1480
1481                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1482                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1483                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1484                         if (s2 == d)
1485                                 M_LXOR(s1, d);
1486                         else {
1487                                 M_INTMOVE(s1, d);
1488                                 M_LXOR(s2, d);
1489                         }
1490                         emit_store(jd, iptr, iptr->dst, d);
1491                         break;
1492
1493                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1494                                       /* val.l = constant                             */
1495
1496                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1497                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1498                         M_INTMOVE(s1, d);
1499                         if (IS_IMM32(iptr->val.l))
1500                                 M_LXOR_IMM(iptr->val.l, d);
1501                         else {
1502                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
1503                                 M_LXOR(REG_ITMP2, d);
1504                         }
1505                         emit_store(jd, iptr, iptr->dst, d);
1506                         break;
1507
1508
1509                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1510                                       /* op1 = variable, val.i = constant             */
1511
1512                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1513                         if (var->flags & INMEMORY) {
1514                                 s1 = REG_ITMP1;
1515                                 M_ILD(s1, REG_SP, var->regoff * 8);
1516                         } else
1517                                 s1 = var->regoff;
1518
1519                         /* Using inc and dec is not faster than add (tested with
1520                            sieve). */
1521
1522                         M_IADD_IMM(iptr->val.i, s1);
1523
1524                         if (var->flags & INMEMORY)
1525                                 M_IST(s1, REG_SP, var->regoff * 8);
1526                         break;
1527
1528
1529                 /* floating operations ************************************************/
1530
1531                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1532
1533                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1534                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1535                         disp = dseg_adds4(cd, 0x80000000);
1536                         M_FLTMOVE(s1, d);
1537                         emit_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1538                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1539                         emit_store(jd, iptr, iptr->dst, d);
1540                         break;
1541
1542                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1543
1544                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1545                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1546                         disp = dseg_adds8(cd, 0x8000000000000000);
1547                         M_FLTMOVE(s1, d);
1548                         emit_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1549                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1550                         emit_store(jd, iptr, iptr->dst, d);
1551                         break;
1552
1553                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1554
1555                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1556                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1557                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1558                         if (s2 == d)
1559                                 M_FADD(s1, d);
1560                         else {
1561                                 M_FLTMOVE(s1, d);
1562                                 M_FADD(s2, d);
1563                         }
1564                         emit_store(jd, iptr, iptr->dst, d);
1565                         break;
1566
1567                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1568
1569                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1570                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1571                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1572                         if (s2 == d)
1573                                 M_DADD(s1, d);
1574                         else {
1575                                 M_FLTMOVE(s1, d);
1576                                 M_DADD(s2, d);
1577                         }
1578                         emit_store(jd, iptr, iptr->dst, d);
1579                         break;
1580
1581                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1582
1583                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1584                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1585                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1586                         if (s2 == d) {
1587                                 M_FLTMOVE(s2, REG_FTMP2);
1588                                 s2 = REG_FTMP2;
1589                         }
1590                         M_FLTMOVE(s1, d);
1591                         M_FSUB(s2, d);
1592                         emit_store(jd, iptr, iptr->dst, d);
1593                         break;
1594
1595                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1596
1597                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1598                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1599                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1600                         if (s2 == d) {
1601                                 M_FLTMOVE(s2, REG_FTMP2);
1602                                 s2 = REG_FTMP2;
1603                         }
1604                         M_FLTMOVE(s1, d);
1605                         M_DSUB(s2, d);
1606                         emit_store(jd, iptr, iptr->dst, d);
1607                         break;
1608
1609                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1610
1611                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1612                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1613                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1614                         if (s2 == d)
1615                                 M_FMUL(s1, d);
1616                         else {
1617                                 M_FLTMOVE(s1, d);
1618                                 M_FMUL(s2, d);
1619                         }
1620                         emit_store(jd, iptr, iptr->dst, d);
1621                         break;
1622
1623                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1624
1625                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1626                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1627                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1628                         if (s2 == d)
1629                                 M_DMUL(s1, d);
1630                         else {
1631                                 M_FLTMOVE(s1, d);
1632                                 M_DMUL(s2, d);
1633                         }
1634                         emit_store(jd, iptr, iptr->dst, d);
1635                         break;
1636
1637                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1638
1639                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1640                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1641                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1642                         if (s2 == d) {
1643                                 M_FLTMOVE(s2, REG_FTMP2);
1644                                 s2 = REG_FTMP2;
1645                         }
1646                         M_FLTMOVE(s1, d);
1647                         M_FDIV(s2, d);
1648                         emit_store(jd, iptr, iptr->dst, d);
1649                         break;
1650
1651                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1652
1653                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1654                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1655                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1656                         if (s2 == d) {
1657                                 M_FLTMOVE(s2, REG_FTMP2);
1658                                 s2 = REG_FTMP2;
1659                         }
1660                         M_FLTMOVE(s1, d);
1661                         M_DDIV(s2, d);
1662                         emit_store(jd, iptr, iptr->dst, d);
1663                         break;
1664
1665                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1666
1667                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1668                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1669                         M_CVTIF(s1, d);
1670                         emit_store(jd, iptr, iptr->dst, d);
1671                         break;
1672
1673                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1674
1675                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1676                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1677                         M_CVTID(s1, d);
1678                         emit_store(jd, iptr, iptr->dst, d);
1679                         break;
1680
1681                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1682
1683                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1684                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1685                         M_CVTLF(s1, d);
1686                         emit_store(jd, iptr, iptr->dst, d);
1687                         break;
1688
1689                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1690
1691                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1692                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1693                         M_CVTLD(s1, d);
1694                         emit_store(jd, iptr, iptr->dst, d);
1695                         break;
1696
1697                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1698
1699                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1700                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1701                         M_CVTFI(s1, d);
1702                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1703                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1704                                 ((REG_RESULT == d) ? 0 : 3);
1705                         M_BNE(disp);
1706                         M_FLTMOVE(s1, REG_FTMP1);
1707                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1708                         M_CALL(REG_ITMP2);
1709                         M_INTMOVE(REG_RESULT, d);
1710                         emit_store(jd, iptr, iptr->dst, d);
1711                         break;
1712
1713                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1714
1715                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1716                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1717                         M_CVTDI(s1, d);
1718                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1719                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1720                                 ((REG_RESULT == d) ? 0 : 3);
1721                         M_BNE(disp);
1722                         M_FLTMOVE(s1, REG_FTMP1);
1723                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1724                         M_CALL(REG_ITMP2);
1725                         M_INTMOVE(REG_RESULT, d);
1726                         emit_store(jd, iptr, iptr->dst, d);
1727                         break;
1728
1729                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1730
1731                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1732                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1733                         M_CVTFL(s1, d);
1734                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1735                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1736                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1737                                 ((REG_RESULT == d) ? 0 : 3);
1738                         M_BNE(disp);
1739                         M_FLTMOVE(s1, REG_FTMP1);
1740                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1741                         M_CALL(REG_ITMP2);
1742                         M_INTMOVE(REG_RESULT, d);
1743                         emit_store(jd, iptr, iptr->dst, d);
1744                         break;
1745
1746                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1747
1748                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1749                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1750                         M_CVTDL(s1, d);
1751                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1752                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1753                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1754                                 ((REG_RESULT == d) ? 0 : 3);
1755                         M_BNE(disp);
1756                         M_FLTMOVE(s1, REG_FTMP1);
1757                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1758                         M_CALL(REG_ITMP2);
1759                         M_INTMOVE(REG_RESULT, d);
1760                         emit_store(jd, iptr, iptr->dst, d);
1761                         break;
1762
1763                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1764
1765                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1766                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1767                         M_CVTFD(s1, d);
1768                         emit_store(jd, iptr, iptr->dst, d);
1769                         break;
1770
1771                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1772
1773                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1774                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1775                         M_CVTDF(s1, d);
1776                         emit_store(jd, iptr, iptr->dst, d);
1777                         break;
1778
1779                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1780                                           /* == => 0, < => 1, > => -1 */
1781
1782                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1783                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1784                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1785                         M_CLR(d);
1786                         M_MOV_IMM(1, REG_ITMP1);
1787                         M_MOV_IMM(-1, REG_ITMP2);
1788                         emit_ucomiss_reg_reg(cd, s1, s2);
1789                         M_CMOVB(REG_ITMP1, d);
1790                         M_CMOVA(REG_ITMP2, d);
1791                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1792                         emit_store(jd, iptr, iptr->dst, d);
1793                         break;
1794
1795                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1796                                           /* == => 0, < => 1, > => -1 */
1797
1798                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1799                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1800                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1801                         M_CLR(d);
1802                         M_MOV_IMM(1, REG_ITMP1);
1803                         M_MOV_IMM(-1, REG_ITMP2);
1804                         emit_ucomiss_reg_reg(cd, s1, s2);
1805                         M_CMOVB(REG_ITMP1, d);
1806                         M_CMOVA(REG_ITMP2, d);
1807                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1808                         emit_store(jd, iptr, iptr->dst, d);
1809                         break;
1810
1811                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1812                                           /* == => 0, < => 1, > => -1 */
1813
1814                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1815                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1816                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1817                         M_CLR(d);
1818                         M_MOV_IMM(1, REG_ITMP1);
1819                         M_MOV_IMM(-1, REG_ITMP2);
1820                         emit_ucomisd_reg_reg(cd, s1, s2);
1821                         M_CMOVB(REG_ITMP1, d);
1822                         M_CMOVA(REG_ITMP2, d);
1823                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1824                         emit_store(jd, iptr, iptr->dst, d);
1825                         break;
1826
1827                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1828                                           /* == => 0, < => 1, > => -1 */
1829
1830                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1831                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1832                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1833                         M_CLR(d);
1834                         M_MOV_IMM(1, REG_ITMP1);
1835                         M_MOV_IMM(-1, REG_ITMP2);
1836                         emit_ucomisd_reg_reg(cd, s1, s2);
1837                         M_CMOVB(REG_ITMP1, d);
1838                         M_CMOVA(REG_ITMP2, d);
1839                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1840                         emit_store(jd, iptr, iptr->dst, d);
1841                         break;
1842
1843
1844                 /* memory operations **************************************************/
1845
1846                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1847
1848                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1849                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1850                         gen_nullptr_check(s1);
1851                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1852                         emit_store(jd, iptr, iptr->dst, d);
1853                         break;
1854
1855                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1856
1857                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1858                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1859                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1860                         if (iptr->op1 == 0) {
1861                                 gen_nullptr_check(s1);
1862                                 gen_bound_check;
1863                         }
1864                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1865                         emit_store(jd, iptr, iptr->dst, d);
1866                         break;
1867
1868                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1869
1870                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1871                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1872                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1873                         if (iptr->op1 == 0) {
1874                                 gen_nullptr_check(s1);
1875                                 gen_bound_check;
1876                         }
1877                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1878                         emit_store(jd, iptr, iptr->dst, d);
1879                         break;
1880
1881                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1882
1883                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1884                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1885                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1886                         if (iptr->op1 == 0) {
1887                                 gen_nullptr_check(s1);
1888                                 gen_bound_check;
1889                         }
1890                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1891                         emit_store(jd, iptr, iptr->dst, d);
1892                         break;
1893
1894                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1895
1896                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1897                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1898                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1899                         if (iptr->op1 == 0) {
1900                                 gen_nullptr_check(s1);
1901                                 gen_bound_check;
1902                         }
1903                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1904                         emit_store(jd, iptr, iptr->dst, d);
1905                         break;
1906
1907                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1908
1909                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1910                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1911                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1912                         if (iptr->op1 == 0) {
1913                                 gen_nullptr_check(s1);
1914                                 gen_bound_check;
1915                         }
1916                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1917                         emit_store(jd, iptr, iptr->dst, d);
1918                         break;
1919
1920                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1921
1922                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1923                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1924                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1925                         if (iptr->op1 == 0) {
1926                                 gen_nullptr_check(s1);
1927                                 gen_bound_check;
1928                         }
1929                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1930                         emit_store(jd, iptr, iptr->dst, d);
1931                         break;
1932
1933                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1934
1935                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1936                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1937                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1938                         if (iptr->op1 == 0) {
1939                                 gen_nullptr_check(s1);
1940                                 gen_bound_check;
1941                         }
1942                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1943                         emit_store(jd, iptr, iptr->dst, d);
1944                         break;
1945
1946                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1947
1948                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1949                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1950                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1951                         if (iptr->op1 == 0) {
1952                                 gen_nullptr_check(s1);
1953                                 gen_bound_check;
1954                         }
1955                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1956                         emit_store(jd, iptr, iptr->dst, d);
1957                         break;
1958
1959
1960                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1961
1962                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
1963                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
1964                         if (iptr->op1 == 0) {
1965                                 gen_nullptr_check(s1);
1966                                 gen_bound_check;
1967                         }
1968                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
1969                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1970                         break;
1971
1972                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1973
1974                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
1975                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
1976                         if (iptr->op1 == 0) {
1977                                 gen_nullptr_check(s1);
1978                                 gen_bound_check;
1979                         }
1980                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
1981                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1982                         break;
1983
1984                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1985
1986                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
1987                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
1988                         if (iptr->op1 == 0) {
1989                                 gen_nullptr_check(s1);
1990                                 gen_bound_check;
1991                         }
1992                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
1993                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1994                         break;
1995
1996                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1997
1998                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
1999                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2000                         if (iptr->op1 == 0) {
2001                                 gen_nullptr_check(s1);
2002                                 gen_bound_check;
2003                         }
2004                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2005                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2006                         break;
2007
2008                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2009
2010                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2011                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2012                         if (iptr->op1 == 0) {
2013                                 gen_nullptr_check(s1);
2014                                 gen_bound_check;
2015                         }
2016                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2017                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2018                         break;
2019
2020                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2021
2022                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2023                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2024                         if (iptr->op1 == 0) {
2025                                 gen_nullptr_check(s1);
2026                                 gen_bound_check;
2027                         }
2028                         s3 = emit_load_s3(jd, iptr, src, REG_FTMP3);
2029                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2030                         break;
2031
2032                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2033
2034                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2035                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2036                         if (iptr->op1 == 0) {
2037                                 gen_nullptr_check(s1);
2038                                 gen_bound_check;
2039                         }
2040                         s3 = emit_load_s3(jd, iptr, src, REG_FTMP3);
2041                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2042                         break;
2043
2044                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2045
2046                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2047                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2048                         if (iptr->op1 == 0) {
2049                                 gen_nullptr_check(s1);
2050                                 gen_bound_check;
2051                         }
2052                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2053
2054                         M_MOV(s1, rd->argintregs[0]);
2055                         M_MOV(s3, rd->argintregs[1]);
2056                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
2057                         M_CALL(REG_ITMP1);
2058                         M_TEST(REG_RESULT);
2059                         M_BEQ(0);
2060                         codegen_add_arraystoreexception_ref(cd);
2061
2062                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2063                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2064                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2065                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2066                         break;
2067
2068
2069                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2070
2071                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2072                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2073                         if (iptr->op1 == 0) {
2074                                 gen_nullptr_check(s1);
2075                                 gen_bound_check;
2076                         }
2077                         emit_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2078                         break;
2079
2080                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2081
2082                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2083                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2084                         if (iptr->op1 == 0) {
2085                                 gen_nullptr_check(s1);
2086                                 gen_bound_check;
2087                         }
2088                         emit_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2089                         break;
2090
2091                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2092
2093                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2094                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2095                         if (iptr->op1 == 0) {
2096                                 gen_nullptr_check(s1);
2097                                 gen_bound_check;
2098                         }
2099                         emit_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2100                         break;
2101
2102                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2103
2104                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2105                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2106                         if (iptr->op1 == 0) {
2107                                 gen_nullptr_check(s1);
2108                                 gen_bound_check;
2109                         }
2110                         emit_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2111                         break;
2112
2113                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2114
2115                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2116                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2117                         if (iptr->op1 == 0) {
2118                                 gen_nullptr_check(s1);
2119                                 gen_bound_check;
2120                         }
2121
2122                         if (IS_IMM32(iptr->val.l)) {
2123                                 emit_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2124                         } else {
2125                                 emit_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2126                                 emit_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2127                         }
2128                         break;
2129
2130                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2131
2132                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2133                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2134                         if (iptr->op1 == 0) {
2135                                 gen_nullptr_check(s1);
2136                                 gen_bound_check;
2137                         }
2138                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2139                         break;
2140
2141
2142                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2143                                       /* op1 = type, val.a = field address            */
2144
2145                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2146                                 disp = dseg_addaddress(cd, NULL);
2147
2148 /*                              PROFILE_CYCLE_STOP; */
2149
2150                                 codegen_addpatchref(cd, PATCHER_get_putstatic,
2151                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), disp);
2152
2153                                 if (opt_showdisassemble) {
2154                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2155                                 }
2156
2157 /*                              PROFILE_CYCLE_START; */
2158
2159                         } else {
2160                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr);
2161
2162                                 disp = dseg_addaddress(cd, &(fi->value));
2163
2164                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2165                                         PROFILE_CYCLE_STOP;
2166
2167                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2168
2169                                         if (opt_showdisassemble) {
2170                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2171                                         }
2172
2173                                         PROFILE_CYCLE_START;
2174                                 }
2175                         }
2176
2177                         /* This approach is much faster than moving the field
2178                            address inline into a register. */
2179
2180                         M_ALD(REG_ITMP1, RIP, -(((ptrint) cd->mcodeptr + 7) -
2181                                                                         (ptrint) cd->mcodebase) + disp);
2182
2183                         switch (iptr->op1) {
2184                         case TYPE_INT:
2185                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
2186                                 M_ILD(d, REG_ITMP1, 0);
2187                                 break;
2188                         case TYPE_LNG:
2189                         case TYPE_ADR:
2190                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
2191                                 M_LLD(d, REG_ITMP1, 0);
2192                                 break;
2193                         case TYPE_FLT:
2194                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2195                                 M_FLD(d, REG_ITMP1, 0);
2196                                 break;
2197                         case TYPE_DBL:
2198                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2199                                 M_DLD(d, REG_ITMP1, 0);
2200                                 break;
2201                         }
2202                         emit_store(jd, iptr, iptr->dst, d);
2203                         break;
2204
2205                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2206                                       /* op1 = type, val.a = field address            */
2207
2208                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2209                                 disp = dseg_addaddress(cd, NULL);
2210
2211 /*                              PROFILE_CYCLE_STOP; */
2212
2213                                 codegen_addpatchref(cd, PATCHER_get_putstatic,
2214                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), disp);
2215
2216                                 if (opt_showdisassemble) {
2217                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2218                                 }
2219
2220 /*                              PROFILE_CYCLE_START; */
2221
2222                         } else {
2223                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr);
2224
2225                                 disp = dseg_addaddress(cd, &(fi->value));
2226
2227                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2228                                         PROFILE_CYCLE_STOP;
2229
2230                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2231
2232                                         if (opt_showdisassemble) {
2233                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2234                                         }
2235
2236                                         PROFILE_CYCLE_START;
2237                                 }
2238                         }
2239
2240                         /* This approach is much faster than moving the field
2241                            address inline into a register. */
2242
2243                         M_ALD(REG_ITMP1, RIP, -(((ptrint) cd->mcodeptr + 7) -
2244                                                                         (ptrint) cd->mcodebase) + disp);
2245
2246                         switch (iptr->op1) {
2247                         case TYPE_INT:
2248                                 s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2249                                 M_IST(s2, REG_ITMP1, 0);
2250                                 break;
2251                         case TYPE_LNG:
2252                         case TYPE_ADR:
2253                                 s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2254                                 M_LST(s2, REG_ITMP1, 0);
2255                                 break;
2256                         case TYPE_FLT:
2257                                 s2 = emit_load_s2(jd, iptr, src, REG_FTMP1);
2258                                 M_FST(s2, REG_ITMP1, 0);
2259                                 break;
2260                         case TYPE_DBL:
2261                                 s2 = emit_load_s2(jd, iptr, src, REG_FTMP1);
2262                                 M_DST(s2, REG_ITMP1, 0);
2263                                 break;
2264                         }
2265                         break;
2266
2267                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2268                                           /* val = value (in current instruction)     */
2269                                           /* op1 = type, val.a = field address (in    */
2270                                           /* following NOP)                           */
2271
2272                         if (INSTRUCTION_IS_UNRESOLVED(iptr + 1)) {
2273                                 disp = dseg_addaddress(cd, NULL);
2274
2275 /*                              PROFILE_CYCLE_STOP; */
2276
2277                                 codegen_addpatchref(cd, PATCHER_get_putstatic,
2278                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr + 1), disp);
2279
2280                                 if (opt_showdisassemble) {
2281                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2282                                 }
2283
2284 /*                              PROFILE_CYCLE_START; */
2285
2286                         } else {
2287                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr + 1);
2288
2289                                 disp = dseg_addaddress(cd, &(fi->value));
2290
2291                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2292                                         PROFILE_CYCLE_STOP;
2293
2294                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2295
2296                                         if (opt_showdisassemble) {
2297                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2298                                         }
2299
2300                                         PROFILE_CYCLE_START;
2301                                 }
2302                         }
2303
2304                         /* This approach is much faster than moving the field
2305                            address inline into a register. */
2306
2307                         M_ALD(REG_ITMP1, RIP, -(((ptrint) cd->mcodeptr + 7) -
2308                                                                         (ptrint) cd->mcodebase) + disp);
2309
2310                         switch (iptr->op1) {
2311                         case TYPE_INT:
2312                         case TYPE_FLT:
2313                                 M_IST_IMM(iptr->val.i, REG_ITMP1, 0);
2314                                 break;
2315                         case TYPE_LNG:
2316                         case TYPE_ADR:
2317                         case TYPE_DBL:
2318                                 if (IS_IMM32(iptr->val.l))
2319                                         M_LST_IMM32(iptr->val.l, REG_ITMP1, 0);
2320                                 else {
2321                                         M_IST_IMM(iptr->val.l, REG_ITMP1, 0);
2322                                         M_IST_IMM(iptr->val.l >> 32, REG_ITMP1, 4);
2323                                 }
2324                                 break;
2325                         }
2326                         break;
2327
2328                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2329                                       /* op1 = type, val.i = field offset             */
2330
2331                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2332                         gen_nullptr_check(s1);
2333
2334                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2335 /*                              PROFILE_CYCLE_STOP; */
2336
2337                                 codegen_addpatchref(cd, PATCHER_get_putfield,
2338                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), 0);
2339
2340                                 if (opt_showdisassemble) {
2341                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2342                                 }
2343
2344 /*                              PROFILE_CYCLE_START; */
2345
2346                                 disp = 0;
2347
2348                         } else
2349                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr)->offset;
2350
2351                         switch (iptr->op1) {
2352                         case TYPE_INT:
2353                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2354                                 M_ILD32(d, s1, disp);
2355                                 break;
2356                         case TYPE_LNG:
2357                         case TYPE_ADR:
2358                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2359                                 M_LLD32(d, s1, disp);
2360                                 break;
2361                         case TYPE_FLT:
2362                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2363                                 M_FLD32(d, s1, disp);
2364                                 break;
2365                         case TYPE_DBL:
2366                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2367                                 M_DLD32(d, s1, disp);
2368                                 break;
2369                         }
2370                         emit_store(jd, iptr, iptr->dst, d);
2371                         break;
2372
2373                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2374                                       /* op1 = type, val.i = field offset             */
2375
2376                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2377                         gen_nullptr_check(s1);
2378
2379                         if (IS_INT_LNG_TYPE(iptr->op1))
2380                                 s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2381                         else
2382                                 s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
2383
2384                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2385 /*                              PROFILE_CYCLE_STOP; */
2386
2387                                 codegen_addpatchref(cd, PATCHER_get_putfield,
2388                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), 0);
2389
2390                                 if (opt_showdisassemble) {
2391                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2392                                 }
2393
2394 /*                              PROFILE_CYCLE_START; */
2395
2396                                 disp = 0;
2397
2398                         } else
2399                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr)->offset;
2400
2401                         switch (iptr->op1) {
2402                         case TYPE_INT:
2403                                 M_IST32(s2, s1, disp);
2404                                 break;
2405                         case TYPE_LNG:
2406                         case TYPE_ADR:
2407                                 M_LST32(s2, s1, disp);
2408                                 break;
2409                         case TYPE_FLT:
2410                                 M_FST32(s2, s1, disp);
2411                                 break;
2412                         case TYPE_DBL:
2413                                 M_DST32(s2, s1, disp);
2414                                 break;
2415                         }
2416                         break;
2417
2418                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2419                                           /* val = value (in current instruction)     */
2420                                           /* op1 = type, val.a = field address (in    */
2421                                           /* following NOP)                           */
2422
2423                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2424                         gen_nullptr_check(s1);
2425
2426                         if (INSTRUCTION_IS_UNRESOLVED(iptr + 1)) {
2427 /*                              PROFILE_CYCLE_STOP; */
2428
2429                                 codegen_addpatchref(cd, PATCHER_putfieldconst,
2430                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr + 1), 0);
2431
2432                                 if (opt_showdisassemble) {
2433                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2434                                 }
2435
2436 /*                              PROFILE_CYCLE_START; */
2437
2438                                 disp = 0;
2439
2440                         } else
2441                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr + 1)->offset;
2442
2443                         switch (iptr->op1) {
2444                         case TYPE_INT:
2445                         case TYPE_FLT:
2446                                 M_IST32_IMM(iptr->val.i, s1, disp);
2447                                 break;
2448                         case TYPE_LNG:
2449                         case TYPE_ADR:
2450                         case TYPE_DBL:
2451                                 M_IST32_IMM(iptr->val.l, s1, disp);
2452                                 M_IST32_IMM(iptr->val.l >> 32, s1, disp + 4);
2453                                 break;
2454                         }
2455                         break;
2456
2457
2458                 /* branch operations **************************************************/
2459
2460                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2461
2462                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2463                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2464
2465                         PROFILE_CYCLE_STOP;
2466
2467 #ifdef ENABLE_VERIFIER
2468                         if (iptr->val.a) {
2469                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2470                                                                         (unresolved_class *) iptr->val.a, 0);
2471
2472                                 if (opt_showdisassemble) {
2473                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2474                                 }
2475                         }
2476 #endif /* ENABLE_VERIFIER */
2477
2478                         M_CALL_IMM(0);                            /* passing exception pc */
2479                         M_POP(REG_ITMP2_XPC);
2480
2481                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2482                         M_JMP(REG_ITMP3);
2483                         break;
2484
2485                 case ICMD_GOTO:         /* ... ==> ...                                */
2486                                         /* op1 = target JavaVM pc                     */
2487
2488                         M_JMP_IMM(0);
2489                         codegen_addreference(cd, (basicblock *) iptr->target);
2490                         break;
2491
2492                 case ICMD_JSR:          /* ... ==> ...                                */
2493                                         /* op1 = target JavaVM pc                     */
2494
2495                         M_CALL_IMM(0);
2496                         codegen_addreference(cd, (basicblock *) iptr->target);
2497                         break;
2498
2499                 case ICMD_RET:          /* ... ==> ...                                */
2500                                         /* op1 = local variable                       */
2501
2502                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2503                         if (var->flags & INMEMORY) {
2504                                 M_ALD(REG_ITMP1, REG_SP, var->regoff * 8);
2505                                 M_JMP(REG_ITMP1);
2506                         } else
2507                                 M_JMP(var->regoff);
2508                         break;
2509
2510                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2511                                         /* op1 = target JavaVM pc                     */
2512
2513                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2514                         M_TEST(s1);
2515                         M_BEQ(0);
2516                         codegen_addreference(cd, (basicblock *) iptr->target);
2517                         break;
2518
2519                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2520                                         /* op1 = target JavaVM pc                     */
2521
2522                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2523                         M_TEST(s1);
2524                         M_BNE(0);
2525                         codegen_addreference(cd, (basicblock *) iptr->target);
2526                         break;
2527
2528                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2529                                         /* op1 = target JavaVM pc, val.i = constant   */
2530
2531                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2532                         M_ICMP_IMM(iptr->val.i, s1);
2533                         M_BEQ(0);
2534                         codegen_addreference(cd, (basicblock *) iptr->target);
2535                         break;
2536
2537                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2538                                         /* op1 = target JavaVM pc, val.i = constant   */
2539
2540                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2541                         M_ICMP_IMM(iptr->val.i, s1);
2542                         M_BLT(0);
2543                         codegen_addreference(cd, (basicblock *) iptr->target);
2544                         break;
2545
2546                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2547                                         /* op1 = target JavaVM pc, val.i = constant   */
2548
2549                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2550                         M_ICMP_IMM(iptr->val.i, s1);
2551                         M_BLE(0);
2552                         codegen_addreference(cd, (basicblock *) iptr->target);
2553                         break;
2554
2555                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2556                                         /* op1 = target JavaVM pc, val.i = constant   */
2557
2558                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2559                         M_ICMP_IMM(iptr->val.i, s1);
2560                         M_BNE(0);
2561                         codegen_addreference(cd, (basicblock *) iptr->target);
2562                         break;
2563
2564                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2565                                         /* op1 = target JavaVM pc, val.i = constant   */
2566
2567                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2568                         M_ICMP_IMM(iptr->val.i, s1);
2569                         M_BGT(0);
2570                         codegen_addreference(cd, (basicblock *) iptr->target);
2571                         break;
2572
2573                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2574                                         /* op1 = target JavaVM pc, val.i = constant   */
2575
2576                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2577                         M_ICMP_IMM(iptr->val.i, s1);
2578                         M_BGE(0);
2579                         codegen_addreference(cd, (basicblock *) iptr->target);
2580                         break;
2581
2582                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2583                                         /* op1 = target JavaVM pc, val.l = constant   */
2584
2585                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2586                         if (IS_IMM32(iptr->val.l))
2587                                 M_LCMP_IMM(iptr->val.l, s1);
2588                         else {
2589                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
2590                                 M_LCMP(REG_ITMP2, s1);
2591                         }
2592                         M_BEQ(0);
2593                         codegen_addreference(cd, (basicblock *) iptr->target);
2594                         break;
2595
2596                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2597                                         /* op1 = target JavaVM pc, val.l = constant   */
2598
2599                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2600                         if (IS_IMM32(iptr->val.l))
2601                                 M_LCMP_IMM(iptr->val.l, s1);
2602                         else {
2603                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
2604                                 M_LCMP(REG_ITMP2, s1);
2605                         }
2606                         M_BLT(0);
2607                         codegen_addreference(cd, (basicblock *) iptr->target);
2608                         break;
2609
2610                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2611                                         /* op1 = target JavaVM pc, val.l = constant   */
2612
2613                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2614                         if (IS_IMM32(iptr->val.l))
2615                                 M_LCMP_IMM(iptr->val.l, s1);
2616                         else {
2617                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
2618                                 M_LCMP(REG_ITMP2, s1);
2619                         }
2620                         M_BLE(0);
2621                         codegen_addreference(cd, (basicblock *) iptr->target);
2622                         break;
2623
2624                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2625                                         /* op1 = target JavaVM pc, val.l = constant   */
2626
2627                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2628                         if (IS_IMM32(iptr->val.l))
2629                                 M_LCMP_IMM(iptr->val.l, s1);
2630                         else {
2631                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
2632                                 M_LCMP(REG_ITMP2, s1);
2633                         }
2634                         M_BNE(0);
2635                         codegen_addreference(cd, (basicblock *) iptr->target);
2636                         break;
2637
2638                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2639                                         /* op1 = target JavaVM pc, val.l = constant   */
2640
2641                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2642                         if (IS_IMM32(iptr->val.l))
2643                                 M_LCMP_IMM(iptr->val.l, s1);
2644                         else {
2645                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
2646                                 M_LCMP(REG_ITMP2, s1);
2647                         }
2648                         M_BGT(0);
2649                         codegen_addreference(cd, (basicblock *) iptr->target);
2650                         break;
2651
2652                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2653                                         /* op1 = target JavaVM pc, val.l = constant   */
2654
2655                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2656                         if (IS_IMM32(iptr->val.l))
2657                                 M_LCMP_IMM(iptr->val.l, s1);
2658                         else {
2659                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
2660                                 M_LCMP(REG_ITMP2, s1);
2661                         }
2662                         M_BGE(0);
2663                         codegen_addreference(cd, (basicblock *) iptr->target);
2664                         break;
2665
2666                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2667                                         /* op1 = target JavaVM pc                     */
2668
2669                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2670                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2671                         M_ICMP(s2, s1);
2672                         M_BEQ(0);
2673                         codegen_addreference(cd, (basicblock *) iptr->target);
2674                         break;
2675
2676                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2677                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2678
2679                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2680                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2681                         M_LCMP(s2, s1);
2682                         M_BEQ(0);
2683                         codegen_addreference(cd, (basicblock *) iptr->target);
2684                         break;
2685
2686                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2687                                         /* op1 = target JavaVM pc                     */
2688
2689                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2690                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2691                         M_ICMP(s2, s1);
2692                         M_BNE(0);
2693                         codegen_addreference(cd, (basicblock *) iptr->target);
2694                         break;
2695
2696                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2697                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2698
2699                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2700                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2701                         M_LCMP(s2, s1);
2702                         M_BNE(0);
2703                         codegen_addreference(cd, (basicblock *) iptr->target);
2704                         break;
2705
2706                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2707                                         /* op1 = target JavaVM pc                     */
2708
2709                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2710                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2711                         M_ICMP(s2, s1);
2712                         M_BLT(0);
2713                         codegen_addreference(cd, (basicblock *) iptr->target);
2714                         break;
2715
2716                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2717                                     /* op1 = target JavaVM pc                     */
2718
2719                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2720                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2721                         M_LCMP(s2, s1);
2722                         M_BLT(0);
2723                         codegen_addreference(cd, (basicblock *) iptr->target);
2724                         break;
2725
2726                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2727                                         /* op1 = target JavaVM pc                     */
2728
2729                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2730                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2731                         M_ICMP(s2, s1);
2732                         M_BGT(0);
2733                         codegen_addreference(cd, (basicblock *) iptr->target);
2734                         break;
2735
2736                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2737                                 /* op1 = target JavaVM pc                     */
2738
2739                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2740                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2741                         M_LCMP(s2, s1);
2742                         M_BGT(0);
2743                         codegen_addreference(cd, (basicblock *) iptr->target);
2744                         break;
2745
2746                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2747                                         /* op1 = target JavaVM pc                     */
2748
2749                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2750                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2751                         M_ICMP(s2, s1);
2752                         M_BLE(0);
2753                         codegen_addreference(cd, (basicblock *) iptr->target);
2754                         break;
2755
2756                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2757                                         /* op1 = target JavaVM pc                     */
2758
2759                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2760                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2761                         M_LCMP(s2, s1);
2762                         M_BLE(0);
2763                         codegen_addreference(cd, (basicblock *) iptr->target);
2764                         break;
2765
2766                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2767                                         /* op1 = target JavaVM pc                     */
2768
2769                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2770                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2771                         M_ICMP(s2, s1);
2772                         M_BGE(0);
2773                         codegen_addreference(cd, (basicblock *) iptr->target);
2774                         break;
2775
2776                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2777                                     /* op1 = target JavaVM pc                     */
2778
2779                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2780                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2781                         M_LCMP(s2, s1);
2782                         M_BGE(0);
2783                         codegen_addreference(cd, (basicblock *) iptr->target);
2784                         break;
2785
2786                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2787
2788                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2789                         break;
2790
2791                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2792                 case ICMD_IFNE_ICONST:  /* val.i = constant                           */
2793                 case ICMD_IFLT_ICONST:
2794                 case ICMD_IFGE_ICONST:
2795                 case ICMD_IFGT_ICONST:
2796                 case ICMD_IFLE_ICONST:
2797
2798                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2799                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2800                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2801                                 if (s1 == d) {
2802                                         M_INTMOVE(s1, REG_ITMP1);
2803                                         s1 = REG_ITMP1;
2804                                 }
2805                                 if (iptr[1].val.i == 0)
2806                                         M_CLR(d);
2807                                 else
2808                                         M_IMOV_IMM(iptr[1].val.i, d);
2809                         }
2810                         if (iptr->val.i == 0)
2811                                 M_CLR(REG_ITMP2);
2812                         else
2813                                 M_IMOV_IMM(iptr->val.i, REG_ITMP2);
2814                         M_ITEST(s1);
2815
2816                         switch (iptr->opc) {
2817                         case ICMD_IFEQ_ICONST:
2818                                 M_CMOVEQ(REG_ITMP2, d);
2819                                 break;
2820                         case ICMD_IFNE_ICONST:
2821                                 M_CMOVNE(REG_ITMP2, d);
2822                                 break;
2823                         case ICMD_IFLT_ICONST:
2824                                 M_CMOVLT(REG_ITMP2, d);
2825                                 break;
2826                         case ICMD_IFGE_ICONST:
2827                                 M_CMOVGE(REG_ITMP2, d);
2828                                 break;
2829                         case ICMD_IFGT_ICONST:
2830                                 M_CMOVGT(REG_ITMP2, d);
2831                                 break;
2832                         case ICMD_IFLE_ICONST:
2833                                 M_CMOVLE(REG_ITMP2, d);
2834                                 break;
2835                         }
2836
2837                         emit_store(jd, iptr, iptr->dst, d);
2838                         break;
2839
2840
2841                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2842                 case ICMD_LRETURN:
2843
2844                         s1 = emit_load_s1(jd, iptr, src, REG_RESULT);
2845                         M_INTMOVE(s1, REG_RESULT);
2846                         goto nowperformreturn;
2847
2848                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2849
2850                         s1 = emit_load_s1(jd, iptr, src, REG_RESULT);
2851                         M_INTMOVE(s1, REG_RESULT);
2852
2853 #ifdef ENABLE_VERIFIER
2854                         if (iptr->val.a) {
2855                                 PROFILE_CYCLE_STOP;
2856
2857                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2858                                                                         (unresolved_class *) iptr->val.a, 0);
2859
2860                                 if (opt_showdisassemble) {
2861                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2862                                 }
2863
2864                                 PROFILE_CYCLE_START;
2865                         }
2866 #endif /* ENABLE_VERIFIER */
2867                         goto nowperformreturn;
2868
2869                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2870                 case ICMD_DRETURN:
2871
2872                         s1 = emit_load_s1(jd, iptr, src, REG_FRESULT);
2873                         M_FLTMOVE(s1, REG_FRESULT);
2874                         goto nowperformreturn;
2875
2876                 case ICMD_RETURN:      /* ...  ==> ...                                */
2877
2878 nowperformreturn:
2879                         {
2880                         s4 i, p;
2881
2882                         p = stackframesize;
2883
2884 #if !defined(NDEBUG)
2885                         /* generate call trace */
2886
2887                         if (opt_verbosecall) {
2888                                 emit_alu_imm_reg(cd, ALU_SUB, 2 * 8, REG_SP);
2889
2890                                 emit_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2891                                 emit_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2892
2893                                 emit_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
2894                                 emit_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2895                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2896                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2897
2898                                 M_MOV_IMM(builtin_displaymethodstop, REG_ITMP1);
2899                                 M_CALL(REG_ITMP1);
2900
2901                                 emit_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2902                                 emit_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2903
2904                                 emit_alu_imm_reg(cd, ALU_ADD, 2 * 8, REG_SP);
2905                         }
2906 #endif /* !defined(NDEBUG) */
2907
2908 #if defined(ENABLE_THREADS)
2909                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2910                                 M_ALD(rd->argintregs[0], REG_SP, rd->memuse * 8);
2911
2912                                 /* we need to save the proper return value */
2913                                 switch (iptr->opc) {
2914                                 case ICMD_IRETURN:
2915                                 case ICMD_ARETURN:
2916                                 case ICMD_LRETURN:
2917                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2918                                         break;
2919                                 case ICMD_FRETURN:
2920                                 case ICMD_DRETURN:
2921                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2922                                         break;
2923                                 }
2924
2925                                 M_MOV_IMM(builtin_monitorexit, REG_ITMP1);
2926                                 M_CALL(REG_ITMP1);
2927
2928                                 /* and now restore the proper return value */
2929                                 switch (iptr->opc) {
2930                                 case ICMD_IRETURN:
2931                                 case ICMD_ARETURN:
2932                                 case ICMD_LRETURN:
2933                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2934                                         break;
2935                                 case ICMD_FRETURN:
2936                                 case ICMD_DRETURN:
2937                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2938                                         break;
2939                                 }
2940                         }
2941 #endif
2942
2943                         /* restore saved registers */
2944
2945                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2946                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2947                         }
2948                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2949                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2950                         }
2951
2952                         /* deallocate stack */
2953
2954                         if (stackframesize)
2955                                 M_AADD_IMM(stackframesize * 8, REG_SP);
2956
2957                         /* generate method profiling code */
2958
2959                         PROFILE_CYCLE_STOP;
2960
2961                         M_RET;
2962                         }
2963                         break;
2964
2965
2966                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2967                         {
2968                                 s4 i, l, *s4ptr;
2969                                 void **tptr;
2970
2971                                 tptr = (void **) iptr->target;
2972
2973                                 s4ptr = iptr->val.a;
2974                                 l = s4ptr[1];                          /* low     */
2975                                 i = s4ptr[2];                          /* high    */
2976
2977                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2978                                 M_INTMOVE(s1, REG_ITMP1);
2979
2980                                 if (l != 0)
2981                                         M_ISUB_IMM(l, REG_ITMP1);
2982
2983                                 i = i - l + 1;
2984
2985                 /* range check */
2986                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2987                                 M_BA(0);
2988
2989                                 codegen_addreference(cd, (basicblock *) tptr[0]);
2990
2991                                 /* build jump table top down and use address of lowest entry */
2992
2993                 /* s4ptr += 3 + i; */
2994                                 tptr += i;
2995
2996                                 while (--i >= 0) {
2997                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
2998                                         --tptr;
2999                                 }
3000
3001                                 /* length of dataseg after last dseg_addtarget is used
3002                                    by load */
3003
3004                                 M_MOV_IMM(0, REG_ITMP2);
3005                                 dseg_adddata(cd);
3006                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
3007                                 M_JMP(REG_ITMP1);
3008                         }
3009                         break;
3010
3011
3012                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
3013                         {
3014                                 s4 i, l, val, *s4ptr;
3015                                 void **tptr;
3016
3017                                 tptr = (void **) iptr->target;
3018
3019                                 s4ptr = iptr->val.a;
3020                                 l = s4ptr[0];                          /* default  */
3021                                 i = s4ptr[1];                          /* count    */
3022
3023                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
3024                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3025
3026                                 while (--i >= 0) {
3027                                         s4ptr += 2;
3028                                         ++tptr;
3029
3030                                         val = s4ptr[0];
3031                                         M_ICMP_IMM(val, s1);
3032                                         M_BEQ(0);
3033                                         codegen_addreference(cd, (basicblock *) tptr[0]);
3034                                 }
3035
3036                                 M_JMP_IMM(0);
3037
3038                                 tptr = (void **) iptr->target;
3039                                 codegen_addreference(cd, (basicblock *) tptr[0]);
3040                         }
3041                         break;
3042
3043
3044                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
3045                                         /* op1 = arg count val.a = builtintable entry */
3046
3047                         bte = iptr->val.a;
3048                         md = bte->md;
3049                         goto gen_method;
3050
3051                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
3052                                         /* op1 = arg count, val.a = method pointer    */
3053
3054                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
3055                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
3056                 case ICMD_INVOKEINTERFACE:
3057
3058                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3059                                 md = INSTRUCTION_UNRESOLVED_METHOD(iptr)->methodref->parseddesc.md;
3060                                 lm = NULL;
3061                         }
3062                         else {
3063                                 lm = INSTRUCTION_RESOLVED_METHODINFO(iptr);
3064                                 md = lm->parseddesc;
3065                         }
3066
3067 gen_method:
3068                         s3 = md->paramcount;
3069
3070                         MCODECHECK((20 * s3) + 128);
3071
3072                         /* copy arguments to registers or stack location */
3073
3074                         for (s3 = s3 - 1; s3 >= 0; s3--, src = src->prev) {
3075                                 if (src->varkind == ARGVAR)
3076                                         continue;
3077                                 if (IS_INT_LNG_TYPE(src->type)) {
3078                                         if (!md->params[s3].inmemory) {
3079                                                 s1 = rd->argintregs[md->params[s3].regoff];
3080                                                 d = emit_load_s1(jd, iptr, src, s1);
3081                                                 M_INTMOVE(d, s1);
3082                                         } else {
3083                                                 d = emit_load_s1(jd, iptr, src, REG_ITMP1);
3084                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
3085                                         }
3086
3087                                 } else {
3088                                         if (!md->params[s3].inmemory) {
3089                                                 s1 = rd->argfltregs[md->params[s3].regoff];
3090                                                 d = emit_load_s1(jd, iptr, src, s1);
3091                                                 M_FLTMOVE(d, s1);
3092                                         } else {
3093                                                 d = emit_load_s1(jd, iptr, src, REG_FTMP1);
3094
3095                                                 if (IS_2_WORD_TYPE(src->type))
3096                                                         M_DST(d, REG_SP, md->params[s3].regoff * 8);
3097                                                 else
3098                                                         M_FST(d, REG_SP, md->params[s3].regoff * 8);
3099                                         }
3100                                 }
3101                         }
3102
3103                         /* generate method profiling code */
3104
3105                         PROFILE_CYCLE_STOP;
3106
3107                         switch (iptr->opc) {
3108                         case ICMD_BUILTIN:
3109                                 a = (ptrint) bte->fp;
3110                                 d = md->returntype.type;
3111
3112                                 M_MOV_IMM(a, REG_ITMP1);
3113                                 M_CALL(REG_ITMP1);
3114
3115                                 /* if op1 == true, we need to check for an exception */
3116
3117                                 if (iptr->op1 == true) {
3118                                         M_TEST(REG_RESULT);
3119                                         M_BEQ(0);
3120                                         codegen_add_fillinstacktrace_ref(cd);
3121                                 }
3122                                 break;
3123
3124                         case ICMD_INVOKESPECIAL:
3125                                 M_TEST(rd->argintregs[0]);
3126                                 M_BEQ(0);
3127                                 codegen_add_nullpointerexception_ref(cd);
3128
3129                                 /* first argument contains pointer */
3130 /*                              gen_nullptr_check(rd->argintregs[0]); */
3131
3132                                 /* access memory for hardware nullptr */
3133 /*                              emit_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); */
3134
3135                                 /* fall through */
3136
3137                         case ICMD_INVOKESTATIC:
3138                                 if (lm == NULL) {
3139                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3140
3141                                         codegen_addpatchref(cd, PATCHER_invokestatic_special,
3142                                                                                 um, 0);
3143
3144                                         if (opt_showdisassemble) {
3145                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3146                                         }
3147
3148                                         a = 0;
3149                                         d = um->methodref->parseddesc.md->returntype.type;
3150
3151                                 } else {
3152                                         a = (ptrint) lm->stubroutine;
3153                                         d = lm->parseddesc->returntype.type;
3154                                 }
3155
3156                                 M_MOV_IMM(a, REG_ITMP2);
3157                                 M_CALL(REG_ITMP2);
3158                                 break;
3159
3160                         case ICMD_INVOKEVIRTUAL:
3161                                 gen_nullptr_check(rd->argintregs[0]);
3162
3163                                 if (lm == NULL) {
3164                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3165
3166                                         codegen_addpatchref(cd, PATCHER_invokevirtual, um, 0);
3167
3168                                         if (opt_showdisassemble) {
3169                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3170                                         }
3171
3172                                         s1 = 0;
3173                                         d = um->methodref->parseddesc.md->returntype.type;
3174
3175                                 } else {
3176                                         s1 = OFFSET(vftbl_t, table[0]) +
3177                                                 sizeof(methodptr) * lm->vftblindex;
3178                                         d = lm->parseddesc->returntype.type;
3179                                 }
3180
3181                                 M_ALD(REG_METHODPTR, rd->argintregs[0],
3182                                           OFFSET(java_objectheader, vftbl));
3183                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
3184                                 M_CALL(REG_ITMP3);
3185                                 break;
3186
3187                         case ICMD_INVOKEINTERFACE:
3188                                 gen_nullptr_check(rd->argintregs[0]);
3189
3190                                 if (lm == NULL) {
3191                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3192
3193                                         codegen_addpatchref(cd, PATCHER_invokeinterface, um, 0);
3194
3195                                         if (opt_showdisassemble) {
3196                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3197                                         }
3198
3199                                         s1 = 0;
3200                                         s2 = 0;
3201                                         d = um->methodref->parseddesc.md->returntype.type;
3202
3203                                 } else {
3204                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3205                                                 sizeof(methodptr) * lm->class->index;
3206
3207                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3208
3209                                         d = lm->parseddesc->returntype.type;
3210                                 }
3211
3212                                 M_ALD(REG_METHODPTR, rd->argintregs[0],
3213                                           OFFSET(java_objectheader, vftbl));
3214                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
3215                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
3216                                 M_CALL(REG_ITMP3);
3217                                 break;
3218                         }
3219
3220                         /* generate method profiling code */
3221
3222                         PROFILE_CYCLE_START;
3223
3224                         /* d contains return type */
3225
3226                         if (d != TYPE_VOID) {
3227                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3228                                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_RESULT);
3229                                         M_INTMOVE(REG_RESULT, s1);
3230                                         emit_store(jd, iptr, iptr->dst, s1);
3231                                 } else {
3232                                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FRESULT);
3233                                         M_FLTMOVE(REG_FRESULT, s1);
3234                                         emit_store(jd, iptr, iptr->dst, s1);
3235                                 }
3236                         }
3237                         break;
3238
3239
3240                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3241
3242                                       /* op1:   0 == array, 1 == class                */
3243                                       /* val.a: (classinfo *) superclass              */
3244
3245                         /*  superclass is an interface:
3246                          *
3247                          *  OK if ((sub == NULL) ||
3248                          *         (sub->vftbl->interfacetablelength > super->index) &&
3249                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3250                          *
3251                          *  superclass is a class:
3252                          *
3253                          *  OK if ((sub == NULL) || (0
3254                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3255                          *         super->vftbl->diffval));
3256                          */
3257
3258                         if (iptr->op1 == 1) {
3259                                 /* object type cast-check */
3260
3261                                 classinfo *super;
3262                                 vftbl_t   *supervftbl;
3263                                 s4         superindex;
3264
3265                                 super = (classinfo *) iptr->val.a;
3266
3267                                 if (!super) {
3268                                         superindex = 0;
3269                                         supervftbl = NULL;
3270
3271                                 } else {
3272                                         superindex = super->index;
3273                                         supervftbl = super->vftbl;
3274                                 }
3275
3276 #if defined(ENABLE_THREADS)
3277                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3278 #endif
3279                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3280
3281                                 /* calculate interface checkcast code size */
3282
3283                                 s2 = 3; /* mov_membase_reg */
3284                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3285
3286                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3287                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3288                                         3 /* test */ + 6 /* jcc */;
3289
3290                                 if (!super)
3291                                         s2 += (opt_showdisassemble ? 5 : 0);
3292
3293                                 /* calculate class checkcast code size */
3294
3295                                 s3 = 3; /* mov_membase_reg */
3296                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3297                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3298
3299 #if 0
3300                                 if (s1 != REG_ITMP1) {
3301                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3302                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3303                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3304                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3305                                         a += 3;    /* sub */
3306
3307                                 } else
3308 #endif
3309                                         {
3310                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3311                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3312                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3313                                         }
3314
3315                                 s3 += 3 /* cmp */ + 6 /* jcc */;
3316
3317                                 if (!super)
3318                                         s3 += (opt_showdisassemble ? 5 : 0);
3319
3320                                 /* if class is not resolved, check which code to call */
3321
3322                                 if (!super) {
3323                                         M_TEST(s1);
3324                                         M_BEQ(6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3325
3326                                         codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3327                                                                                 (constant_classref *) iptr->target, 0);
3328
3329                                         if (opt_showdisassemble) {
3330                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3331                                         }
3332
3333                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
3334                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
3335                                         M_BEQ(s2 + 5);
3336                                 }
3337
3338                                 /* interface checkcast code */
3339
3340                                 if (!super || (super->flags & ACC_INTERFACE)) {
3341                                         if (super) {
3342                                                 M_TEST(s1);
3343                                                 M_BEQ(s2);
3344                                         }
3345
3346                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3347
3348                                         if (!super) {
3349                                                 codegen_addpatchref(cd,
3350                                                                                         PATCHER_checkcast_instanceof_interface,
3351                                                                                         (constant_classref *) iptr->target,
3352                                                                                         0);
3353
3354                                                 if (opt_showdisassemble) {
3355                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3356                                                 }
3357                                         }
3358
3359                                         emit_movl_membase32_reg(cd, REG_ITMP2,
3360                                                                                           OFFSET(vftbl_t, interfacetablelength),
3361                                                                                           REG_ITMP3);
3362                                         /* XXX TWISTI: should this be int arithmetic? */
3363                                         M_LSUB_IMM32(superindex, REG_ITMP3);
3364                                         M_TEST(REG_ITMP3);
3365                                         M_BLE(0);
3366                                         codegen_add_classcastexception_ref(cd);
3367                                         emit_mov_membase32_reg(cd, REG_ITMP2,
3368                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
3369                                                                                          superindex * sizeof(methodptr*),
3370                                                                                          REG_ITMP3);
3371                                         M_TEST(REG_ITMP3);
3372                                         M_BEQ(0);
3373                                         codegen_add_classcastexception_ref(cd);
3374
3375                                         if (!super)
3376                                                 M_JMP_IMM(s3);
3377                                 }
3378
3379                                 /* class checkcast code */
3380
3381                                 if (!super || !(super->flags & ACC_INTERFACE)) {
3382                                         if (super) {
3383                                                 M_TEST(s1);
3384                                                 M_BEQ(s3);
3385                                         }
3386
3387                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3388
3389                                         if (!super) {
3390                                                 codegen_addpatchref(cd, PATCHER_checkcast_class,
3391                                                                                         (constant_classref *) iptr->target,
3392                                                                                         0);
3393
3394                                                 if (opt_showdisassemble) {
3395                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3396                                                 }
3397                                         }
3398
3399                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3400 #if defined(ENABLE_THREADS)
3401                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3402 #endif
3403                                         emit_movl_membase32_reg(cd, REG_ITMP2,
3404                                                                                           OFFSET(vftbl_t, baseval),
3405                                                                                           REG_ITMP2);
3406                                         /*                                      if (s1 != REG_ITMP1) { */
3407                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3408                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
3409                                         /*                                                                                              REG_ITMP1); */
3410                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3411                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3412                                         /*                                                                                              REG_ITMP3); */
3413                                         /*  #if defined(ENABLE_THREADS) */
3414                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3415                                         /*  #endif */
3416                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
3417
3418                                         /*                                      } else { */
3419                                         emit_movl_membase32_reg(cd, REG_ITMP3,
3420                                                                                           OFFSET(vftbl_t, baseval),
3421                                                                                           REG_ITMP3);
3422                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3423                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3424                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3425                                         /*                                      } */
3426 #if defined(ENABLE_THREADS)
3427                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3428 #endif
3429                                         M_LCMP(REG_ITMP3, REG_ITMP2);
3430                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3431                                         codegen_add_classcastexception_ref(cd);
3432                                 }
3433                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
3434
3435                         } else {
3436                                 /* array type cast-check */
3437
3438                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3439                                 M_INTMOVE(s1, rd->argintregs[0]);
3440
3441                                 if (iptr->val.a == NULL) {
3442                                         codegen_addpatchref(cd, PATCHER_builtin_arraycheckcast,
3443                                                                                 (constant_classref *) iptr->target, 0);
3444
3445                                         if (opt_showdisassemble) {
3446                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3447                                         }
3448                                 }
3449
3450                                 M_MOV_IMM(iptr->val.a, rd->argintregs[1]);
3451                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
3452                                 M_CALL(REG_ITMP1);
3453                                 M_TEST(REG_RESULT);
3454                                 M_BEQ(0);
3455                                 codegen_add_classcastexception_ref(cd);
3456
3457                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3458                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
3459                         }
3460                         M_INTMOVE(s1, d);
3461                         emit_store(jd, iptr, iptr->dst, d);
3462                         break;
3463
3464                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3465
3466                                       /* op1:   0 == array, 1 == class                */
3467                                       /* val.a: (classinfo *) superclass              */
3468
3469                         /*  superclass is an interface:
3470                          *
3471                          *  return (sub != NULL) &&
3472                          *         (sub->vftbl->interfacetablelength > super->index) &&
3473                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3474                          *
3475                          *  superclass is a class:
3476                          *
3477                          *  return ((sub != NULL) && (0
3478                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3479                          *          super->vftbl->diffvall));
3480                          */
3481
3482                         {
3483                         classinfo *super;
3484                         vftbl_t   *supervftbl;
3485                         s4         superindex;
3486
3487                         super = (classinfo *) iptr->val.a;
3488
3489                         if (!super) {
3490                                 superindex = 0;
3491                                 supervftbl = NULL;
3492
3493                         } else {
3494                                 superindex = super->index;
3495                                 supervftbl = super->vftbl;
3496                         }
3497
3498 #if defined(ENABLE_THREADS)
3499             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3500 #endif
3501
3502                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3503                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
3504                         if (s1 == d) {
3505                                 M_INTMOVE(s1, REG_ITMP1);
3506                                 s1 = REG_ITMP1;
3507                         }
3508
3509                         /* calculate interface instanceof code size */
3510
3511                         s2 = 3; /* mov_membase_reg */
3512                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3513                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3514                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3515                                 3 /* test */ + 4 /* setcc */;
3516
3517                         if (!super)
3518                                 s2 += (opt_showdisassemble ? 5 : 0);
3519
3520                         /* calculate class instanceof code size */
3521
3522                         s3 = 3; /* mov_membase_reg */
3523                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3524                         s3 += 10; /* mov_imm_reg */
3525                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3526                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3527                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3528                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3529                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3530                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3531                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3532
3533                         if (!super)
3534                                 s3 += (opt_showdisassemble ? 5 : 0);
3535
3536                         emit_alu_reg_reg(cd, ALU_XOR, d, d);
3537
3538                         /* if class is not resolved, check which code to call */
3539
3540                         if (!super) {
3541                                 emit_test_reg_reg(cd, s1, s1);
3542                                 emit_jcc(cd, CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3543                                                                                          7 + 6 + s2 + 5 + s3));
3544
3545                                 codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3546                                                                         (constant_classref *) iptr->target, 0);
3547
3548                                 if (opt_showdisassemble) {
3549                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3550                                 }
3551
3552                                 emit_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3553                                 emit_alul_imm_reg(cd, ALU_AND, ACC_INTERFACE, REG_ITMP3);
3554                                 emit_jcc(cd, CC_Z, s2 + 5);
3555                         }
3556
3557                         /* interface instanceof code */
3558
3559                         if (!super || (super->flags & ACC_INTERFACE)) {
3560                                 if (super) {
3561                                         emit_test_reg_reg(cd, s1, s1);
3562                                         emit_jcc(cd, CC_Z, s2);
3563                                 }
3564
3565                                 emit_mov_membase_reg(cd, s1,
3566                                                                            OFFSET(java_objectheader, vftbl),
3567                                                                            REG_ITMP1);
3568                                 if (!super) {
3569                                         codegen_addpatchref(cd,
3570                                                                                 PATCHER_checkcast_instanceof_interface,
3571                                                                                 (constant_classref *) iptr->target, 0);
3572
3573                                         if (opt_showdisassemble) {
3574                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3575                                         }
3576                                 }
3577
3578                                 emit_movl_membase32_reg(cd, REG_ITMP1,
3579                                                                                   OFFSET(vftbl_t, interfacetablelength),
3580                                                                                   REG_ITMP3);
3581                                 emit_alu_imm32_reg(cd, ALU_SUB, superindex, REG_ITMP3);
3582                                 emit_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3583
3584                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3585
3586                                 emit_jcc(cd, CC_LE, a);
3587                                 emit_mov_membase32_reg(cd, REG_ITMP1,
3588                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3589                                                                                  superindex * sizeof(methodptr*),
3590                                                                                  REG_ITMP1);
3591                                 emit_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3592                                 emit_setcc_reg(cd, CC_NE, d);
3593
3594                                 if (!super)
3595                                         emit_jmp_imm(cd, s3);
3596                         }
3597
3598                         /* class instanceof code */
3599
3600                         if (!super || !(super->flags & ACC_INTERFACE)) {
3601                                 if (super) {
3602                                         emit_test_reg_reg(cd, s1, s1);
3603                                         emit_jcc(cd, CC_E, s3);
3604                                 }
3605
3606                                 emit_mov_membase_reg(cd, s1,
3607                                                                            OFFSET(java_objectheader, vftbl),
3608                                                                            REG_ITMP1);
3609
3610                                 if (!super) {
3611                                         codegen_addpatchref(cd, PATCHER_instanceof_class,
3612                                                                                 (constant_classref *) iptr->target, 0);
3613
3614                                         if (opt_showdisassemble) {
3615                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3616                                         }
3617                                 }
3618
3619                                 emit_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3620 #if defined(ENABLE_THREADS)
3621                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3622 #endif
3623                                 emit_movl_membase_reg(cd, REG_ITMP1,
3624                                                                                 OFFSET(vftbl_t, baseval),
3625                                                                                 REG_ITMP1);
3626                                 emit_movl_membase_reg(cd, REG_ITMP2,
3627                                                                                 OFFSET(vftbl_t, diffval),
3628                                                                                 REG_ITMP3);
3629                                 emit_movl_membase_reg(cd, REG_ITMP2,
3630                                                                                 OFFSET(vftbl_t, baseval),
3631                                                                                 REG_ITMP2);
3632 #if defined(ENABLE_THREADS)
3633                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3634 #endif
3635                                 emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
3636                                 emit_alu_reg_reg(cd, ALU_XOR, d, d); /* may be REG_ITMP2 */
3637                                 emit_alu_reg_reg(cd, ALU_CMP, REG_ITMP3, REG_ITMP1);
3638                                 emit_setcc_reg(cd, CC_BE, d);
3639                         }
3640                         emit_store(jd, iptr, iptr->dst, d);
3641                         }
3642                         break;
3643
3644                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3645                                          /* op1 = dimension, val.a = class            */
3646
3647                         /* check for negative sizes and copy sizes to stack if necessary  */
3648
3649                         MCODECHECK((10 * 4 * iptr->op1) + 5 + 10 * 8);
3650
3651                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3652                                 /* copy SAVEDVAR sizes to stack */
3653
3654                                 if (src->varkind != ARGVAR) {
3655                                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP1);
3656                                         M_LST(s2, REG_SP, s1 * 8);
3657                                 }
3658                         }
3659
3660                         /* is a patcher function set? */
3661
3662                         if (iptr->val.a == NULL) {
3663                                 codegen_addpatchref(cd, PATCHER_builtin_multianewarray,
3664                                                                         (constant_classref *) iptr->target, 0);
3665
3666                                 if (opt_showdisassemble) {
3667                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3668                                 }
3669
3670                                 a = 0;
3671
3672                         } else {
3673                                 a = (ptrint) iptr->val.a;
3674                         }
3675
3676                         /* a0 = dimension count */
3677
3678                         M_MOV_IMM(iptr->op1, rd->argintregs[0]);
3679
3680                         /* a1 = arrayvftbl */
3681
3682                         M_MOV_IMM(iptr->val.a, rd->argintregs[1]);
3683
3684                         /* a2 = pointer to dimensions = stack pointer */
3685
3686                         M_MOV(REG_SP, rd->argintregs[2]);
3687
3688                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3689                         M_CALL(REG_ITMP1);
3690
3691                         /* check for exception before result assignment */
3692
3693                         M_TEST(REG_RESULT);
3694                         M_BEQ(0);
3695                         codegen_add_fillinstacktrace_ref(cd);
3696
3697                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_RESULT);
3698                         M_INTMOVE(REG_RESULT, s1);
3699                         emit_store(jd, iptr, iptr->dst, s1);
3700                         break;
3701
3702                 default:
3703                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3704                         return false;
3705         } /* switch */
3706
3707         } /* for instruction */
3708
3709         /* copy values to interface registers */
3710
3711         src = bptr->outstack;
3712         len = bptr->outdepth;
3713         MCODECHECK(512);
3714 #if defined(ENABLE_LSRA)
3715         if (!opt_lsra)
3716 #endif
3717         while (src) {
3718                 len--;
3719                 if ((src->varkind != STACKVAR)) {
3720                         s2 = src->type;
3721                         if (IS_FLT_DBL_TYPE(s2)) {
3722                                 s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
3723                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3724                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3725
3726                                 } else {
3727                                         emit_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3728                                 }
3729
3730                         } else {
3731                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3732                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3733                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3734
3735                                 } else {
3736                                         emit_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3737                                 }
3738                         }
3739                 }
3740                 src = src->prev;
3741         }
3742
3743         /* At the end of a basic block we may have to append some nops,
3744            because the patcher stub calling code might be longer than the
3745            actual instruction. So codepatching does not change the
3746            following block unintentionally. */
3747
3748         if (cd->mcodeptr < cd->lastmcodeptr) {
3749                 while (cd->mcodeptr < cd->lastmcodeptr) {
3750                         M_NOP;
3751                 }
3752         }
3753
3754         } /* if (bptr -> flags >= BBREACHED) */
3755         } /* for basic block */
3756
3757         dseg_createlinenumbertable(cd);
3758
3759
3760         /* generate exception and patcher stubs */
3761
3762         {
3763                 exceptionref *eref;
3764                 patchref     *pref;
3765                 ptrint        mcode;
3766                 u1           *savedmcodeptr;
3767                 u1           *tmpmcodeptr;
3768
3769                 savedmcodeptr = NULL;
3770
3771                 /* generate exception stubs */
3772
3773                 for (eref = cd->exceptionrefs; eref != NULL; eref = eref->next) {
3774                         gen_resolvebranch(cd->mcodebase + eref->branchpos,
3775                                                           eref->branchpos,
3776                                                           cd->mcodeptr - cd->mcodebase);
3777
3778                         MCODECHECK(512);
3779
3780                         /* Check if the exception is an
3781                            ArrayIndexOutOfBoundsException.  If so, move index register
3782                            into a4. */
3783
3784                         if (eref->reg != -1)
3785                                 M_MOV(eref->reg, rd->argintregs[4]);
3786
3787                         /* calcuate exception address */
3788
3789                         M_MOV_IMM(0, rd->argintregs[3]);
3790                         dseg_adddata(cd);
3791                         M_AADD_IMM32(eref->branchpos - 6, rd->argintregs[3]);
3792
3793                         /* move function to call into REG_ITMP3 */
3794
3795                         M_MOV_IMM(eref->function, REG_ITMP3);
3796
3797                         if (savedmcodeptr != NULL) {
3798                                 M_JMP_IMM(savedmcodeptr - cd->mcodeptr - 5);
3799
3800                         } else {
3801                                 savedmcodeptr = cd->mcodeptr;
3802
3803                                 emit_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3804                                 M_MOV(REG_SP, rd->argintregs[1]);
3805                                 M_ALD(rd->argintregs[2], REG_SP, stackframesize * 8);
3806
3807                                 M_ASUB_IMM(2 * 8, REG_SP);
3808                                 M_AST(rd->argintregs[3], REG_SP, 0 * 8);         /* store XPC */
3809
3810                                 M_CALL(REG_ITMP3);
3811
3812                                 M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3813                                 M_AADD_IMM(2 * 8, REG_SP);
3814
3815                                 M_MOV_IMM(asm_handle_exception, REG_ITMP3);
3816                                 M_JMP(REG_ITMP3);
3817                         }
3818                 }
3819
3820
3821                 /* generate code patching stub call code */
3822
3823                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
3824                         /* check size of code segment */
3825
3826                         MCODECHECK(512);
3827
3828                         /* Get machine code which is patched back in later. A
3829                            `call rel32' is 5 bytes long (but read 8 bytes). */
3830
3831                         savedmcodeptr = cd->mcodebase + pref->branchpos;
3832                         mcode = *((ptrint *) savedmcodeptr);
3833
3834                         /* patch in `call rel32' to call the following code */
3835
3836                         tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr          */
3837                         cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position */
3838
3839                         M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
3840
3841                         cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr   */
3842
3843                         /* move pointer to java_objectheader onto stack */
3844
3845 #if defined(ENABLE_THREADS)
3846                         /* create a virtual java_objectheader */
3847
3848                         (void) dseg_addaddress(cd, NULL);                         /* flcword    */
3849                         (void) dseg_addaddress(cd, lock_get_initial_lock_word()); /* monitorPtr */
3850                         a = dseg_addaddress(cd, NULL);                            /* vftbl      */
3851
3852                         emit_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP3);
3853                         M_PUSH(REG_ITMP3);
3854 #else
3855                         M_PUSH_IMM(0);
3856 #endif
3857
3858                         /* move machine code bytes and classinfo pointer into registers */
3859
3860                         M_MOV_IMM(mcode, REG_ITMP3);
3861                         M_PUSH(REG_ITMP3);
3862                         M_MOV_IMM(pref->ref, REG_ITMP3);
3863                         M_PUSH(REG_ITMP3);
3864                         M_MOV_IMM(pref->disp, REG_ITMP3);
3865                         M_PUSH(REG_ITMP3);
3866
3867                         M_MOV_IMM(pref->patcher, REG_ITMP3);
3868                         M_PUSH(REG_ITMP3);
3869
3870                         M_MOV_IMM(asm_wrapper_patcher, REG_ITMP3);
3871                         M_JMP(REG_ITMP3);
3872                 }
3873         }
3874
3875         /* generate replacement-out stubs */
3876
3877         {
3878                 int i;
3879
3880                 replacementpoint = jd->code->rplpoints;
3881
3882                 for (i = 0; i < jd->code->rplpointcount; ++i, ++replacementpoint) {
3883                         /* check code segment size */
3884
3885                         MCODECHECK(512);
3886
3887                         /* note start of stub code */
3888
3889                         replacementpoint->outcode = (u1*) (ptrint)(cd->mcodeptr - cd->mcodebase);
3890
3891                         /* make machine code for patching */
3892
3893                         disp = (ptrint)(replacementpoint->outcode - replacementpoint->pc) - 5;
3894                         replacementpoint->mcode = 0xe9 | ((u8)disp << 8);
3895
3896                         /* push address of `rplpoint` struct */
3897
3898                         M_MOV_IMM(replacementpoint, REG_ITMP3);
3899                         M_PUSH(REG_ITMP3);
3900
3901                         /* jump to replacement function */
3902
3903                         M_MOV_IMM(asm_replacement_out, REG_ITMP3);
3904                         M_JMP(REG_ITMP3);
3905                 }
3906         }
3907
3908         codegen_finish(jd);
3909
3910         /* everything's ok */
3911
3912         return true;
3913 }
3914
3915
3916 /* createcompilerstub **********************************************************
3917
3918    Creates a stub routine which calls the compiler.
3919
3920 *******************************************************************************/
3921
3922 #define COMPILERSTUB_DATASIZE    2 * SIZEOF_VOID_P
3923 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
3924
3925 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
3926
3927
3928 u1 *createcompilerstub(methodinfo *m)
3929 {
3930         u1          *s;                     /* memory to hold the stub            */
3931         ptrint      *d;
3932         codegendata *cd;
3933         s4           dumpsize;
3934
3935         s = CNEW(u1, COMPILERSTUB_SIZE);
3936
3937         /* set data pointer and code pointer */
3938
3939         d = (ptrint *) s;
3940         s = s + COMPILERSTUB_DATASIZE;
3941
3942         /* mark start of dump memory area */
3943
3944         dumpsize = dump_size();
3945
3946         cd = DNEW(codegendata);
3947         cd->mcodeptr = s;
3948
3949         /* Store the methodinfo* in the same place as in the methodheader
3950            for compiled methods. */
3951
3952         d[0] = (ptrint) asm_call_jit_compiler;
3953         d[1] = (ptrint) m;
3954
3955         /* code for the stub */
3956
3957         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 1 * SIZEOF_VOID_P));       /* methodinfo  */
3958         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 2 * SIZEOF_VOID_P));  /* compiler pointer */
3959         M_JMP(REG_ITMP3);
3960
3961 #if defined(ENABLE_STATISTICS)
3962         if (opt_stat)
3963                 count_cstub_len += COMPILERSTUB_SIZE;
3964 #endif
3965
3966         /* release dump area */
3967
3968         dump_release(dumpsize);
3969
3970         return s;
3971 }
3972
3973
3974 /* createnativestub ************************************************************
3975
3976    Creates a stub routine which calls a native method.
3977
3978 *******************************************************************************/
3979
3980 u1 *createnativestub(functionptr f, jitdata *jd, methoddesc *nmd)
3981 {
3982         methodinfo   *m;
3983         codegendata  *cd;
3984         registerdata *rd;
3985         methoddesc   *md;
3986         s4            stackframesize;       /* size of stackframe if needed       */
3987         s4            nativeparams;
3988         s4            i, j;                 /* count variables                    */
3989         s4            t;
3990         s4            s1, s2;
3991
3992         /* get required compiler data */
3993
3994         m  = jd->m;
3995         cd = jd->cd;
3996         rd = jd->rd;
3997
3998         /* initialize variables */
3999
4000         md = m->parseddesc;
4001         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
4002
4003         /* calculate stack frame size */
4004
4005         stackframesize =
4006                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
4007                 sizeof(localref_table) / SIZEOF_VOID_P +
4008                 INT_ARG_CNT + FLT_ARG_CNT + 1 +         /* + 1 for function address   */
4009                 nmd->memuse;
4010
4011         if (!(stackframesize & 0x1))                /* keep stack 16-byte aligned */
4012                 stackframesize++;
4013
4014         /* create method header */
4015
4016         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
4017         (void) dseg_adds4(cd, stackframesize * 8);              /* FrameSize      */
4018         (void) dseg_adds4(cd, 0);                               /* IsSync         */
4019         (void) dseg_adds4(cd, 0);                               /* IsLeaf         */
4020         (void) dseg_adds4(cd, 0);                               /* IntSave        */
4021         (void) dseg_adds4(cd, 0);                               /* FltSave        */
4022         (void) dseg_addlinenumbertablesize(cd);
4023         (void) dseg_adds4(cd, 0);                               /* ExTableSize    */
4024
4025         /* generate native method profiling code */
4026
4027         if (opt_prof) {
4028                 /* count frequency */
4029
4030                 M_MOV_IMM(m, REG_ITMP2);
4031                 M_IINC_MEMBASE(REG_ITMP2, OFFSET(methodinfo, frequency));
4032         }
4033
4034         /* generate stub code */
4035
4036         M_ASUB_IMM(stackframesize * 8, REG_SP);
4037
4038 #if !defined(NDEBUG)
4039         /* generate call trace */
4040
4041         if (opt_verbosecall) {
4042                 /* save integer and float argument registers */
4043
4044                 for (i = 0, j = 1; i < md->paramcount; i++) {
4045                         if (!md->params[i].inmemory) {
4046                                 s1 = md->params[i].regoff;
4047
4048                                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4049                                         M_LST(rd->argintregs[s1], REG_SP, j * 8);
4050                                 else
4051                                         M_DST(rd->argfltregs[s1], REG_SP, j * 8);
4052
4053                                 j++;
4054                         }
4055                 }
4056
4057                 /* show integer hex code for float arguments */
4058
4059                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++) {
4060                         /* if the paramtype is a float, we have to right shift all
4061                            following integer registers */
4062
4063                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type)) {
4064                                 for (s1 = INT_ARG_CNT - 2; s1 >= i; s1--)
4065                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
4066
4067                                 emit_movd_freg_reg(cd, rd->argfltregs[j], rd->argintregs[i]);
4068                                 j++;
4069                         }
4070                 }
4071
4072                 M_MOV_IMM(m, REG_ITMP1);
4073                 M_AST(REG_ITMP1, REG_SP, 0 * 8);
4074                 M_MOV_IMM(builtin_trace_args, REG_ITMP1);
4075                 M_CALL(REG_ITMP1);
4076
4077                 /* restore integer and float argument registers */
4078
4079                 for (i = 0, j = 1; i < md->paramcount; i++) {
4080                         if (!md->params[i].inmemory) {
4081                                 s1 = md->params[i].regoff;
4082
4083                                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4084                                         M_LLD(rd->argintregs[s1], REG_SP, j * 8);
4085                                 else
4086                                         M_DLD(rd->argfltregs[s1], REG_SP, j * 8);
4087
4088                                 j++;
4089                         }
4090                 }
4091         }
4092 #endif /* !defined(NDEBUG) */
4093
4094         /* get function address (this must happen before the stackframeinfo) */
4095
4096 #if !defined(WITH_STATIC_CLASSPATH)
4097         if (f == NULL) {
4098                 codegen_addpatchref(cd, PATCHER_resolve_native, m, 0);
4099
4100                 if (opt_showdisassemble) {
4101                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
4102                 }
4103         }
4104 #endif
4105
4106         M_MOV_IMM(f, REG_ITMP3);
4107
4108
4109         /* save integer and float argument registers */
4110
4111         for (i = 0, j = 0; i < md->paramcount; i++) {
4112                 if (!md->params[i].inmemory) {
4113                         s1 = md->params[i].regoff;
4114
4115                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4116                                 M_LST(rd->argintregs[s1], REG_SP, j * 8);
4117                         else
4118                                 M_DST(rd->argfltregs[s1], REG_SP, j * 8);
4119
4120                         j++;
4121                 }
4122         }
4123
4124         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4125
4126         /* create dynamic stack info */
4127
4128         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4129         emit_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[1]);
4130         M_ALEA(REG_SP, stackframesize * 8 + SIZEOF_VOID_P, rd->argintregs[2]);
4131         M_ALD(rd->argintregs[3], REG_SP, stackframesize * 8);
4132         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
4133         M_CALL(REG_ITMP1);
4134
4135         /* restore integer and float argument registers */
4136
4137         for (i = 0, j = 0; i < md->paramcount; i++) {
4138                 if (!md->params[i].inmemory) {
4139                         s1 = md->params[i].regoff;
4140
4141                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4142                                 M_LLD(rd->argintregs[s1], REG_SP, j * 8);
4143                         else
4144                                 M_DLD(rd->argfltregs[s1], REG_SP, j * 8);
4145
4146                         j++;
4147                 }
4148         }
4149
4150         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4151
4152
4153         /* copy or spill arguments to new locations */
4154
4155         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
4156                 t = md->paramtypes[i].type;
4157
4158                 if (IS_INT_LNG_TYPE(t)) {
4159                         if (!md->params[i].inmemory) {
4160                                 s1 = rd->argintregs[md->params[i].regoff];
4161
4162                                 if (!nmd->params[j].inmemory) {
4163                                         s2 = rd->argintregs[nmd->params[j].regoff];
4164                                         M_INTMOVE(s1, s2);
4165
4166                                 } else {
4167                                         s2 = nmd->params[j].regoff;
4168                                         M_LST(s1, REG_SP, s2 * 8);
4169                                 }
4170
4171                         } else {
4172                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4173                                 s2 = nmd->params[j].regoff;
4174                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
4175                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
4176                         }
4177
4178                 } else {
4179                         /* We only copy spilled float arguments, as the float argument    */
4180                         /* registers keep unchanged.                                      */
4181
4182                         if (md->params[i].inmemory) {
4183                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4184                                 s2 = nmd->params[j].regoff;
4185
4186                                 if (IS_2_WORD_TYPE(t)) {
4187                                         M_DLD(REG_FTMP1, REG_SP, s1 * 8);
4188                                         M_DST(REG_FTMP1, REG_SP, s2 * 8);
4189                                 } else {
4190                                         M_FLD(REG_FTMP1, REG_SP, s1 * 8);
4191                                         M_FST(REG_FTMP1, REG_SP, s2 * 8);
4192                                 }
4193                         }
4194                 }
4195         }
4196
4197         /* put class into second argument register */
4198
4199         if (m->flags & ACC_STATIC)
4200                 M_MOV_IMM(m->class, rd->argintregs[1]);
4201
4202         /* put env into first argument register */
4203
4204         M_MOV_IMM(_Jv_env, rd->argintregs[0]);
4205
4206         /* do the native function call */
4207
4208         M_CALL(REG_ITMP3);
4209
4210         /* save return value */
4211
4212         if (md->returntype.type != TYPE_VOID) {
4213                 if (IS_INT_LNG_TYPE(md->returntype.type))
4214                         M_LST(REG_RESULT, REG_SP, 0 * 8);
4215                 else
4216                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
4217         }
4218
4219         /* remove native stackframe info */
4220
4221         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4222         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
4223         M_CALL(REG_ITMP1);
4224
4225 #if !defined(NDEBUG)
4226         /* generate call trace */
4227
4228         if (opt_verbosecall) {
4229                 /* just restore the value we need, don't care about the other */
4230
4231                 if (md->returntype.type != TYPE_VOID) {
4232                         if (IS_INT_LNG_TYPE(md->returntype.type))
4233                                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
4234                         else
4235                                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4236                 }
4237
4238                 M_MOV_IMM(m, rd->argintregs[0]);
4239                 M_MOV(REG_RESULT, rd->argintregs[1]);
4240                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4241                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4242
4243                 M_MOV_IMM(builtin_displaymethodstop, REG_ITMP1);
4244                 M_CALL(REG_ITMP1);
4245         }
4246 #endif /* !defined(NDEBUG) */
4247
4248         /* check for exception */
4249
4250 #if defined(ENABLE_THREADS)
4251         M_MOV_IMM(builtin_get_exceptionptrptr, REG_ITMP3);
4252         M_CALL(REG_ITMP3);
4253 #else
4254         M_MOV_IMM(&_no_threads_exceptionptr, REG_RESULT);
4255 #endif
4256         M_ALD(REG_ITMP2, REG_RESULT, 0);
4257
4258         /* restore return value */
4259
4260         if (md->returntype.type != TYPE_VOID) {
4261                 if (IS_INT_LNG_TYPE(md->returntype.type))
4262                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
4263                 else
4264                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4265         }
4266
4267         /* test for exception */
4268
4269         M_TEST(REG_ITMP2);
4270         M_BNE(7 + 1);
4271
4272         /* remove stackframe */
4273
4274         M_AADD_IMM(stackframesize * 8, REG_SP);
4275         M_RET;
4276
4277
4278         /* handle exception */
4279
4280 #if defined(ENABLE_THREADS)
4281         M_LST(REG_ITMP2, REG_SP, 0 * 8);
4282         M_MOV_IMM(builtin_get_exceptionptrptr, REG_ITMP3);
4283         M_CALL(REG_ITMP3);
4284         M_AST_IMM32(0, REG_RESULT, 0);                 /* clear exception pointer */
4285         M_LLD(REG_ITMP1_XPTR, REG_SP, 0 * 8);
4286 #else
4287         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
4288         M_MOV_IMM(&_no_threads_exceptionptr, REG_ITMP3);
4289         M_AST_IMM32(0, REG_ITMP3, 0);                  /* clear exception pointer */
4290 #endif
4291
4292         /* remove stackframe */
4293
4294         M_AADD_IMM(stackframesize * 8, REG_SP);
4295
4296         M_LLD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
4297         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
4298
4299         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
4300         M_JMP(REG_ITMP3);
4301
4302
4303         /* process patcher calls **************************************************/
4304
4305         {
4306                 patchref *pref;
4307                 ptrint    mcode;
4308                 u1       *savedmcodeptr;
4309                 u1       *tmpmcodeptr;
4310 #if defined(ENABLE_THREADS)
4311                 s4        disp;
4312 #endif
4313
4314                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4315                         /* Get machine code which is patched back in later. A
4316                            `call rel32' is 5 bytes long (but read 8 bytes). */
4317
4318                         savedmcodeptr = cd->mcodebase + pref->branchpos;
4319                         mcode = *((ptrint *) savedmcodeptr);
4320
4321                         /* patch in `call rel32' to call the following code               */
4322
4323                         tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr          */
4324                         cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position */
4325
4326                         M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
4327
4328                         cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr   */
4329
4330                         /* move pointer to java_objectheader onto stack */
4331
4332 #if defined(ENABLE_THREADS)
4333                         /* create a virtual java_objectheader */
4334
4335                         (void) dseg_addaddress(cd, NULL);                         /* flcword    */
4336                         (void) dseg_addaddress(cd, lock_get_initial_lock_word()); /* monitorPtr */
4337                         disp = dseg_addaddress(cd, NULL);                         /* vftbl      */
4338
4339                         emit_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP3);
4340                         M_PUSH(REG_ITMP3);
4341 #else
4342                         M_PUSH_IMM(0);
4343 #endif
4344
4345                         /* move machine code bytes and classinfo pointer into registers */
4346
4347                         M_MOV_IMM(mcode, REG_ITMP3);
4348                         M_PUSH(REG_ITMP3);
4349                         M_MOV_IMM(pref->ref, REG_ITMP3);
4350                         M_PUSH(REG_ITMP3);
4351                         M_MOV_IMM(pref->disp, REG_ITMP3);
4352                         M_PUSH(REG_ITMP3);
4353
4354                         M_MOV_IMM(pref->patcher, REG_ITMP3);
4355                         M_PUSH(REG_ITMP3);
4356
4357                         M_MOV_IMM(asm_wrapper_patcher, REG_ITMP3);
4358                         M_JMP(REG_ITMP3);
4359                 }
4360         }
4361
4362         codegen_finish(jd);
4363
4364         return jd->code->entrypoint;
4365 }
4366
4367
4368 /*
4369  * These are local overrides for various environment variables in Emacs.
4370  * Please do not remove this and leave it at the end of the file, where
4371  * Emacs will automagically detect them.
4372  * ---------------------------------------------------------------------
4373  * Local variables:
4374  * mode: c
4375  * indent-tabs-mode: t
4376  * c-basic-offset: 4
4377  * tab-width: 4
4378  * End:
4379  * vim:noexpandtab:sw=4:ts=4:
4380  */