src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31             Edwin Steiner
  32
  33    $Id: codegen.c 5123 2006-07-12 21:45:34Z twisti $
  34
  35 */
  36
  37
  38 #include "config.h"
  39
  40 #include <assert.h>
  41 #include <stdio.h>
  42
  43 #include "vm/types.h"
  44
  45 #include "md.h"
  46 #include "md-abi.h"
  47
  48 #include "vm/jit/x86_64/arch.h"
  49 #include "vm/jit/x86_64/codegen.h"
  50 #include "vm/jit/x86_64/md-emit.h"
  51
  52 #include "mm/memory.h"
  53 #include "native/jni.h"
  54 #include "native/native.h"
  55
  56 #if defined(ENABLE_THREADS)
  57 # include "threads/native/lock.h"
  58 #endif
  59
  60 #include "vm/builtin.h"
  61 #include "vm/exceptions.h"
  62 #include "vm/global.h"
  63 #include "vm/loader.h"
  64 #include "vm/options.h"
  65 #include "vm/statistics.h"
  66 #include "vm/stringlocal.h"
  67 #include "vm/vm.h"
  68 #include "vm/jit/asmpart.h"
  69 #include "vm/jit/codegen-common.h"
  70 #include "vm/jit/dseg.h"
  71 #include "vm/jit/emit.h"
  72 #include "vm/jit/jit.h"
  73 #include "vm/jit/methodheader.h"
  74 #include "vm/jit/parse.h"
  75 #include "vm/jit/patcher.h"
  76 #include "vm/jit/reg.h"
  77 #include "vm/jit/replace.h"
  78
  79 #if defined(ENABLE_LSRA)
  80 # include "vm/jit/allocator/lsra.h"
  81 #endif
  82
  83
  84 /* codegen *********************************************************************
  85
  86    Generates machine code.
  87
  88 *******************************************************************************/
  89
  90 bool codegen(jitdata *jd)
  91 {
  92         methodinfo         *m;
  93         codeinfo           *code;
  94         codegendata        *cd;
  95         registerdata       *rd;
  96         s4                  len, s1, s2, s3, d, disp;
  97         u2                  currentline;
  98         ptrint              a;
  99         s4                  stackframesize;
 100         stackptr            src;
 101         varinfo            *var;
 102         basicblock         *bptr;
 103         instruction        *iptr;
 104         exceptiontable     *ex;
 105         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 106         builtintable_entry *bte;
 107         methoddesc         *md;
 108         rplpoint           *replacementpoint;
 109
 110         /* get required compiler data */
 111
 112         m    = jd->m;
 113         code = jd->code;
 114         cd   = jd->cd;
 115         rd   = jd->rd;
 116
 117         /* prevent compiler warnings */
 118
 119         d = 0;
 120         lm = NULL;
 121         bte = NULL;
 122
 123         {
 124         s4 i, p, t, l;
 125         s4 savedregs_num;
 126
 127         savedregs_num = 0;
 128
 129         /* space to save used callee saved registers */
 130
 131         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 132         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 133
 134         stackframesize = rd->memuse + savedregs_num;
 135
 136 #if defined(ENABLE_THREADS)
 137         /* space to save argument of monitor_enter */
 138
 139         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 140                 stackframesize++;
 141 #endif
 142
 143         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 144            native code e.g. libc or jni (alignment problems with
 145            movaps). */
 146
 147         if (!jd->isleafmethod || opt_verbosecall)
 148                 stackframesize |= 0x1;
 149
 150         /* create method header */
 151
 152         (void) dseg_addaddress(cd, code);                      /* CodeinfoPointer */
 153         (void) dseg_adds4(cd, stackframesize * 8);             /* FrameSize       */
 154
 155 #if defined(ENABLE_THREADS)
 156         /* IsSync contains the offset relative to the stack pointer for the
 157            argument of monitor_exit used in the exception handler. Since the
 158            offset could be zero and give a wrong meaning of the flag it is
 159            offset by one.
 160         */
 161
 162         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 163                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);       /* IsSync          */
 164         else
 165 #endif
 166                 (void) dseg_adds4(cd, 0);                          /* IsSync          */
 167
 168         (void) dseg_adds4(cd, jd->isleafmethod);               /* IsLeaf          */
 169         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave         */
 170         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave         */
 171
 172         (void) dseg_addlinenumbertablesize(cd);
 173
 174         (void) dseg_adds4(cd, cd->exceptiontablelength);       /* ExTableSize     */
 175
 176         /* create exception table */
 177
 178         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 179                 dseg_addtarget(cd, ex->start);
 180                 dseg_addtarget(cd, ex->end);
 181                 dseg_addtarget(cd, ex->handler);
 182                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 183         }
 184
 185         /* generate method profiling code */
 186
 187         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 188                 /* count frequency */
 189
 190                 M_MOV_IMM(code, REG_ITMP3);
 191                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 192
 193                 PROFILE_CYCLE_START;
 194         }
 195
 196         /* create stack frame (if necessary) */
 197
 198         if (stackframesize)
 199                 M_ASUB_IMM(stackframesize * 8, REG_SP);
 200
 201         /* save used callee saved registers */
 202
 203         p = stackframesize;
 204         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 205                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 206         }
 207         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 208                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 209         }
 210
 211         /* take arguments out of register or stack frame */
 212
 213         md = m->parseddesc;
 214
 215         for (p = 0, l = 0; p < md->paramcount; p++) {
 216                 t = md->paramtypes[p].type;
 217                 var = &(rd->locals[l][t]);
 218                 l++;
 219                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 220                         l++;
 221                 if (var->type < 0)
 222                         continue;
 223                 s1 = md->params[p].regoff;
 224                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 225                         s2 = rd->argintregs[s1];
 226                         if (!md->params[p].inmemory) {           /* register arguments    */
 227                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 228                                         M_INTMOVE(s2, var->regoff);
 229
 230                                 } else {                             /* reg arg -> spilled    */
 231                                     M_LST(s2, REG_SP, var->regoff * 8);
 232                                 }
 233
 234                         } else {                                 /* stack arguments       */
 235                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 236                                         /* + 8 for return address */
 237                                         M_LLD(var->regoff, REG_SP, (stackframesize + s1) * 8 + 8);
 238
 239                                 } else {                             /* stack arg -> spilled  */
 240                                         var->regoff = stackframesize + s1 + 1;
 241                                 }
 242                         }
 243
 244                 } else {                                     /* floating args         */
 245                         if (!md->params[p].inmemory) {           /* register arguments    */
 246                                 s2 = rd->argfltregs[s1];
 247                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 248                                         M_FLTMOVE(s2, var->regoff);
 249
 250                                 } else {                                         /* reg arg -> spilled    */
 251                                         M_DST(s2, REG_SP, var->regoff * 8);
 252                                 }
 253
 254                         } else {                                 /* stack arguments       */
 255                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 256                                         M_DLD(var->regoff, REG_SP, (stackframesize + s1) * 8 + 8);
 257
 258                                 } else {
 259                                         var->regoff = stackframesize + s1 + 1;
 260                                 }
 261                         }
 262                 }
 263         }  /* end for */
 264
 265         /* save monitorenter argument */
 266
 267 #if defined(ENABLE_THREADS)
 268         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 269                 /* stack offset for monitor argument */
 270
 271                 s1 = rd->memuse;
 272
 273                 if (opt_verbosecall) {
 274                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 275
 276                         for (p = 0; p < INT_ARG_CNT; p++)
 277                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 278
 279                         for (p = 0; p < FLT_ARG_CNT; p++)
 280                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 281
 282                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 283                 }
 284
 285                 /* decide which monitor enter function to call */
 286
 287                 if (m->flags & ACC_STATIC) {
 288                         M_MOV_IMM(&m->class->object.header, rd->argintregs[0]);
 289                 }
 290                 else {
 291                         M_TEST(rd->argintregs[0]);
 292                         M_BEQ(0);
 293                         codegen_add_nullpointerexception_ref(cd);
 294                 }
 295
 296                 M_AST(rd->argintregs[0], REG_SP, s1 * 8);
 297                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 298                 M_CALL(REG_ITMP1);
 299
 300                 if (opt_verbosecall) {
 301                         for (p = 0; p < INT_ARG_CNT; p++)
 302                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 303
 304                         for (p = 0; p < FLT_ARG_CNT; p++)
 305                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 306
 307                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 308                 }
 309         }
 310 #endif
 311
 312 #if !defined(NDEBUG)
 313         /* Copy argument registers to stack and call trace function with
 314            pointer to arguments on stack. */
 315
 316         if (opt_verbosecall) {
 317                 M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 318
 319                 /* save integer argument registers */
 320
 321                 for (p = 0; p < INT_ARG_CNT; p++)
 322                         M_LST(rd->argintregs[p], REG_SP, (1 + p) * 8);
 323
 324                 /* save float argument registers */
 325
 326                 for (p = 0; p < FLT_ARG_CNT; p++)
 327                         M_DST(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 328
 329                 /* save temporary registers for leaf methods */
 330
 331                 if (jd->isleafmethod) {
 332                         for (p = 0; p < INT_TMP_CNT; p++)
 333                                 M_LST(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 334
 335                         for (p = 0; p < FLT_TMP_CNT; p++)
 336                                 M_DST(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 337                 }
 338
 339                 /* show integer hex code for float arguments */
 340
 341                 for (p = 0, l = 0; p < md->paramcount && p < INT_ARG_CNT; p++) {
 342                         /* If the paramtype is a float, we have to right shift all
 343                            following integer registers. */
 344
 345                         if (IS_FLT_DBL_TYPE(md->paramtypes[p].type)) {
 346                                 for (s1 = INT_ARG_CNT - 2; s1 >= p; s1--)
 347                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
 348
 349                                 emit_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 350                                 l++;
 351                         }
 352                 }
 353
 354                 M_MOV_IMM(m, REG_ITMP2);
 355                 M_AST(REG_ITMP2, REG_SP, 0 * 8);
 356                 M_MOV_IMM(builtin_trace_args, REG_ITMP1);
 357                 M_CALL(REG_ITMP1);
 358
 359                 /* restore integer argument registers */
 360
 361                 for (p = 0; p < INT_ARG_CNT; p++)
 362                         M_LLD(rd->argintregs[p], REG_SP, (1 + p) * 8);
 363
 364                 /* restore float argument registers */
 365
 366                 for (p = 0; p < FLT_ARG_CNT; p++)
 367                         M_DLD(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 368
 369                 /* restore temporary registers for leaf methods */
 370
 371                 if (jd->isleafmethod) {
 372                         for (p = 0; p < INT_TMP_CNT; p++)
 373                                 M_LLD(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 374
 375                         for (p = 0; p < FLT_TMP_CNT; p++)
 376                                 M_DLD(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 377                 }
 378
 379                 M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 380         }
 381 #endif /* !defined(NDEBUG) */
 382
 383         }
 384
 385         /* end of header generation */
 386
 387         replacementpoint = jd->code->rplpoints;
 388
 389         /* walk through all basic blocks */
 390
 391         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 392
 393                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 394
 395                 if (bptr->flags >= BBREACHED) {
 396
 397                         /* branch resolving */
 398
 399                         branchref *bref;
 400                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 401                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 402                                                                   bref->branchpos,
 403                                                                   bptr->mpc);
 404                         }
 405
 406                 /* handle replacement points */
 407
 408                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 409                         replacementpoint->pc = (u1*)(ptrint)bptr->mpc; /* will be resolved later */
 410
 411                         replacementpoint++;
 412
 413                         assert(cd->lastmcodeptr <= cd->mcodeptr);
 414                         cd->lastmcodeptr = cd->mcodeptr + 5; /* 5 byte jmp patch */
 415                 }
 416
 417                 /* copy interface registers to their destination */
 418
 419                 src = bptr->instack;
 420                 len = bptr->indepth;
 421                 MCODECHECK(512);
 422
 423 #if 0
 424                 /* generate basicblock profiling code */
 425
 426                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 427                         /* count frequency */
 428
 429                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 430                         M_IINC_MEMBASE(REG_ITMP3, bptr->debug_nr * 4);
 431
 432                         /* if this is an exception handler, start profiling again */
 433
 434                         if (bptr->type == BBTYPE_EXH)
 435                                 PROFILE_CYCLE_START;
 436                 }
 437 #endif
 438
 439 #if defined(ENABLE_LSRA)
 440                 if (opt_lsra) {
 441                         while (src != NULL) {
 442                                 len--;
 443                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 444                                         if (bptr->type == BBTYPE_SBR) {
 445 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 446                                                 if (!(src->flags & INMEMORY))
 447                                                         d = src->regoff;
 448                                                 else
 449                                                         d = REG_ITMP1;
 450                                                 M_POP(d);
 451                                                 emit_store(jd, NULL, src, d);
 452
 453                                         } else if (bptr->type == BBTYPE_EXH) {
 454 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 455                                                 if (!(src->flags & INMEMORY))
 456                                                         d= src->regoff;
 457                                                 else
 458                                                         d=REG_ITMP1;
 459                                                 M_INTMOVE(REG_ITMP1, d);
 460                                                 emit_store(jd, NULL, src, d);
 461                                         }
 462                                 }
 463                                 src = src->prev;
 464                         }
 465
 466                 } else {
 467 #endif
 468
 469                 while (src != NULL) {
 470                         len--;
 471                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 472                                 if (bptr->type == BBTYPE_SBR) {
 473                                         d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 474                                         M_POP(d);
 475                                         emit_store(jd, NULL, src, d);
 476
 477                                 } else if (bptr->type == BBTYPE_EXH) {
 478                                         d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 479                                         M_INTMOVE(REG_ITMP1, d);
 480                                         emit_store(jd, NULL, src, d);
 481                                 }
 482
 483                         } else {
 484                                 d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 485                                 if ((src->varkind != STACKVAR)) {
 486                                         s2 = src->type;
 487                                         if (IS_FLT_DBL_TYPE(s2)) {
 488                                                 s1 = rd->interfaces[len][s2].regoff;
 489
 490                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 491                                                         M_FLTMOVE(s1, d);
 492                                                 else
 493                                                         M_DLD(d, REG_SP, s1 * 8);
 494
 495                                                 emit_store(jd, NULL, src, d);
 496
 497                                         } else {
 498                                                 s1 = rd->interfaces[len][s2].regoff;
 499
 500                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 501                                                         M_INTMOVE(s1, d);
 502                                                 else
 503                                                         M_LLD(d, REG_SP, s1 * 8);
 504
 505                                                 emit_store(jd, NULL, src, d);
 506                                         }
 507                                 }
 508                         }
 509                         src = src->prev;
 510                 }
 511 #if defined(ENABLE_LSRA)
 512                 }
 513 #endif
 514                 /* walk through all instructions */
 515
 516                 src = bptr->instack;
 517                 len = bptr->icount;
 518                 currentline = 0;
 519
 520                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 521                         if (iptr->line != currentline) {
 522                                 dseg_addlinenumber(cd, iptr->line);
 523                                 currentline = iptr->line;
 524                         }
 525
 526                         MCODECHECK(1024);                         /* 1KB should be enough */
 527
 528                         switch (iptr->opc) {
 529                         case ICMD_INLINE_START: /* internal ICMDs                         */
 530                         case ICMD_INLINE_END:
 531                                 break;
 532
 533                         case ICMD_NOP:    /* ...  ==> ...                                 */
 534                                 break;
 535
 536                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 537
 538                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 539                                 M_TEST(s1);
 540                                 M_BEQ(0);
 541                                 codegen_add_nullpointerexception_ref(cd);
 542                                 break;
 543
 544                 /* constant operations ************************************************/
 545
 546                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 547                                       /* op1 = 0, val.i = constant                    */
 548
 549                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 550                         ICONST(d, iptr->val.i);
 551                         emit_store(jd, iptr, iptr->dst, d);
 552                         break;
 553
 554                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 555                                       /* op1 = 0, val.l = constant                    */
 556
 557                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 558                         LCONST(d, iptr->val.l);
 559                         emit_store(jd, iptr, iptr->dst, d);
 560                         break;
 561
 562                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 563                                       /* op1 = 0, val.f = constant                    */
 564
 565                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 566                         disp = dseg_addfloat(cd, iptr->val.f);
 567                         emit_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + disp, d);
 568                         emit_store(jd, iptr, iptr->dst, d);
 569                         break;
 570
 571                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 572                                       /* op1 = 0, val.d = constant                    */
 573
 574                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 575                         disp = dseg_adddouble(cd, iptr->val.d);
 576                         emit_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, d);
 577                         emit_store(jd, iptr, iptr->dst, d);
 578                         break;
 579
 580                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 581                                       /* op1 = 0, val.a = constant                    */
 582
 583                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 584
 585                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 586 /*                              PROFILE_CYCLE_STOP; */
 587
 588                                 codegen_addpatchref(cd, PATCHER_aconst,
 589                                                                         ICMD_ACONST_UNRESOLVED_CLASSREF(iptr), 0);
 590
 591                                 if (opt_showdisassemble) {
 592                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
 593                                 }
 594
 595 /*                              PROFILE_CYCLE_START; */
 596
 597                                 M_MOV_IMM(NULL, d);
 598
 599                         } else {
 600                                 if (iptr->val.a == 0)
 601                                         M_CLR(d);
 602                                 else
 603                                         M_MOV_IMM(iptr->val.a, d);
 604                         }
 605                         emit_store(jd, iptr, iptr->dst, d);
 606                         break;
 607
 608
 609                 /* load/store operations **********************************************/
 610
 611                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 612                                       /* op1 = local variable                         */
 613
 614                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 615                         if ((iptr->dst->varkind == LOCALVAR) &&
 616                                 (iptr->dst->varnum == iptr->op1))
 617                                 break;
 618                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 619                         if (var->flags & INMEMORY)
 620                                 M_ILD(d, REG_SP, var->regoff * 8);
 621                         else
 622                                 M_INTMOVE(var->regoff, d);
 623                         emit_store(jd, iptr, iptr->dst, d);
 624                         break;
 625
 626                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 627                 case ICMD_ALOAD:      /* op1 = local variable                         */
 628
 629                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 630                         if ((iptr->dst->varkind == LOCALVAR) &&
 631                                 (iptr->dst->varnum == iptr->op1))
 632                                 break;
 633                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 634                         if (var->flags & INMEMORY)
 635                                 M_LLD(d, REG_SP, var->regoff * 8);
 636                         else
 637                                 M_INTMOVE(var->regoff, d);
 638                         emit_store(jd, iptr, iptr->dst, d);
 639                         break;
 640
 641                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 642                                       /* op1 = local variable                         */
 643
 644                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 645                         if ((iptr->dst->varkind == LOCALVAR) &&
 646                             (iptr->dst->varnum == iptr->op1))
 647                                 break;
 648                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 649                         if (var->flags & INMEMORY)
 650                                 M_FLD(d, REG_SP, var->regoff * 8);
 651                         else
 652                                 M_FLTMOVE(var->regoff, d);
 653                         emit_store(jd, iptr, iptr->dst, d);
 654                         break;
 655
 656                 case ICMD_DLOAD:      /* ...  ==> ..., content of local variable      */
 657                                       /* op1 = local variable                         */
 658
 659                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 660                         if ((iptr->dst->varkind == LOCALVAR) &&
 661                             (iptr->dst->varnum == iptr->op1))
 662                                 break;
 663                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 664                         if (var->flags & INMEMORY)
 665                                 M_DLD(d, REG_SP, var->regoff * 8);
 666                         else
 667                                 M_FLTMOVE(var->regoff, d);
 668                         emit_store(jd, iptr, iptr->dst, d);
 669                         break;
 670
 671                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 672                                       /* op1 = local variable                         */
 673
 674                         if ((src->varkind == LOCALVAR) && (src->varnum == iptr->op1))
 675                                 break;
 676                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 677                         if (var->flags & INMEMORY) {
 678                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 679                                 M_IST(s1, REG_SP, var->regoff * 8);
 680                         } else {
 681                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 682                                 M_INTMOVE(s1, var->regoff);
 683                         }
 684                         break;
 685
 686                 case ICMD_LSTORE:     /* ..., value  ==> ...                          */
 687                 case ICMD_ASTORE:     /* op1 = local variable                         */
 688
 689                         if ((src->varkind == LOCALVAR) && (src->varnum == iptr->op1))
 690                                 break;
 691                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 692                         if (var->flags & INMEMORY) {
 693                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 694                                 M_LST(s1, REG_SP, var->regoff * 8);
 695                         } else {
 696                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 697                                 M_INTMOVE(s1, var->regoff);
 698                         }
 699                         break;
 700
 701                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 702                                       /* op1 = local variable                         */
 703
 704                         if ((src->varkind == LOCALVAR) &&
 705                             (src->varnum == iptr->op1)) {
 706                                 break;
 707                         }
 708                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 709                         if (var->flags & INMEMORY) {
 710                                 s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
 711                                 M_FST(s1, REG_SP, var->regoff * 8);
 712                         } else {
 713                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 714                                 M_FLTMOVE(s1, var->regoff);
 715                         }
 716                         break;
 717
 718                 case ICMD_DSTORE:     /* ..., value  ==> ...                          */
 719                                       /* op1 = local variable                         */
 720
 721                         if ((src->varkind == LOCALVAR) &&
 722                             (src->varnum == iptr->op1)) {
 723                                 break;
 724                         }
 725                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 726                         if (var->flags & INMEMORY) {
 727                                 s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
 728                                 M_DST(s1, REG_SP, var->regoff * 8);
 729                         } else {
 730                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 731                                 M_FLTMOVE(s1, var->regoff);
 732                         }
 733                         break;
 734
 735
 736                 /* pop/dup/swap operations ********************************************/
 737
 738                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 739
 740                 case ICMD_POP:        /* ..., value  ==> ...                          */
 741                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 742                         break;
 743
 744                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 745                         M_COPY(src, iptr->dst);
 746                         break;
 747
 748                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 749
 750                         M_COPY(src,       iptr->dst);
 751                         M_COPY(src->prev, iptr->dst->prev);
 752                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 753                         break;
 754
 755                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 756
 757                         M_COPY(src,             iptr->dst);
 758                         M_COPY(src->prev,       iptr->dst->prev);
 759                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 760                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 761                         break;
 762
 763                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 764
 765                         M_COPY(src,       iptr->dst);
 766                         M_COPY(src->prev, iptr->dst->prev);
 767                         break;
 768
 769                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 770
 771                         M_COPY(src,             iptr->dst);
 772                         M_COPY(src->prev,       iptr->dst->prev);
 773                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 774                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 775                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 776                         break;
 777
 778                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 779
 780                         M_COPY(src,                   iptr->dst);
 781                         M_COPY(src->prev,             iptr->dst->prev);
 782                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 783                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 784                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 785                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 786                         break;
 787
 788                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 789
 790                         M_COPY(src,       iptr->dst->prev);
 791                         M_COPY(src->prev, iptr->dst);
 792                         break;
 793
 794
 795                 /* integer operations *************************************************/
 796
 797                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 798
 799                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 800                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 801                         M_INTMOVE(s1, d);
 802                         M_INEG(d);
 803                         emit_store(jd, iptr, iptr->dst, d);
 804                         break;
 805
 806                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 807
 808                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 809                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 810                         M_INTMOVE(s1, d);
 811                         M_LNEG(d);
 812                         emit_store(jd, iptr, iptr->dst, d);
 813                         break;
 814
 815                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 816
 817                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 818                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 819                         M_ISEXT(s1, d);
 820                         emit_store(jd, iptr, iptr->dst, d);
 821                         break;
 822
 823                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 824
 825                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 826                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 827                         M_IMOV(s1, d);
 828                         emit_store(jd, iptr, iptr->dst, d);
 829                         break;
 830
 831                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 832
 833                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 834                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 835                         M_BSEXT(s1, d);
 836                         emit_store(jd, iptr, iptr->dst, d);
 837                         break;
 838
 839                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 840
 841                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 842                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 843                         M_CZEXT(s1, d);
 844                         emit_store(jd, iptr, iptr->dst, d);
 845                         break;
 846
 847                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 848
 849                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 850                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 851                         M_SSEXT(s1, d);
 852                         emit_store(jd, iptr, iptr->dst, d);
 853                         break;
 854
 855
 856                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 857
 858                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 859                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 860                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 861                         if (s2 == d)
 862                                 M_IADD(s1, d);
 863                         else {
 864                                 M_INTMOVE(s1, d);
 865                                 M_IADD(s2, d);
 866                         }
 867                         emit_store(jd, iptr, iptr->dst, d);
 868                         break;
 869
 870                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 871                                       /* val.i = constant                             */
 872
 873                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 874                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 875                         M_INTMOVE(s1, d);
 876                         M_IADD_IMM(iptr->val.i, d);
 877                         emit_store(jd, iptr, iptr->dst, d);
 878                         break;
 879
 880                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 881
 882                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 883                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 884                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 885                         if (s2 == d)
 886                                 M_LADD(s1, d);
 887                         else {
 888                                 M_INTMOVE(s1, d);
 889                                 M_LADD(s2, d);
 890                         }
 891                         emit_store(jd, iptr, iptr->dst, d);
 892                         break;
 893
 894                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 895                                       /* val.l = constant                             */
 896
 897                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 898                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 899                         M_INTMOVE(s1, d);
 900                         if (IS_IMM32(iptr->val.l))
 901                                 M_LADD_IMM(iptr->val.l, d);
 902                         else {
 903                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
 904                                 M_LADD(REG_ITMP2, d);
 905                         }
 906                         emit_store(jd, iptr, iptr->dst, d);
 907                         break;
 908
 909                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 910
 911                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 912                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 913                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 914                         if (s2 == d) {
 915                                 M_INTMOVE(s1, REG_ITMP1);
 916                                 M_ISUB(s2, REG_ITMP1);
 917                                 M_INTMOVE(REG_ITMP1, d);
 918                         } else {
 919                                 M_INTMOVE(s1, d);
 920                                 M_ISUB(s2, d);
 921                         }
 922                         emit_store(jd, iptr, iptr->dst, d);
 923                         break;
 924
 925                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 926                                       /* val.i = constant                             */
 927
 928                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 929                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 930                         M_INTMOVE(s1, d);
 931                         M_ISUB_IMM(iptr->val.i, d);
 932                         emit_store(jd, iptr, iptr->dst, d);
 933                         break;
 934
 935                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 936
 937                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 938                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 939                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 940                         if (s2 == d) {
 941                                 M_INTMOVE(s1, REG_ITMP1);
 942                                 M_LSUB(s2, REG_ITMP1);
 943                                 M_INTMOVE(REG_ITMP1, d);
 944                         } else {
 945                                 M_INTMOVE(s1, d);
 946                                 M_LSUB(s2, d);
 947                         }
 948                         emit_store(jd, iptr, iptr->dst, d);
 949                         break;
 950
 951                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 952                                       /* val.l = constant                             */
 953
 954                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 955                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 956                         M_INTMOVE(s1, d);
 957                         if (IS_IMM32(iptr->val.l))
 958                                 M_LSUB_IMM(iptr->val.l, d);
 959                         else {
 960                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
 961                                 M_LSUB(REG_ITMP2, d);
 962                         }
 963                         emit_store(jd, iptr, iptr->dst, d);
 964                         break;
 965
 966                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 967
 968                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 969                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 970                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 971                         if (s2 == d)
 972                                 M_IMUL(s1, d);
 973                         else {
 974                                 M_INTMOVE(s1, d);
 975                                 M_IMUL(s2, d);
 976                         }
 977                         emit_store(jd, iptr, iptr->dst, d);
 978                         break;
 979
 980                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 981                                       /* val.i = constant                             */
 982
 983                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 984                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 985                         if (iptr->val.i == 2) {
 986                                 M_INTMOVE(s1, d);
 987                                 M_ISLL_IMM(1, d);
 988                         } else
 989                                 M_IMUL_IMM(s1, iptr->val.i, d);
 990                         emit_store(jd, iptr, iptr->dst, d);
 991                         break;
 992
 993                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 994
 995                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 996                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 997                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 998                         if (s2 == d)
 999                                 M_LMUL(s1, d);
1000                         else {
1001                                 M_INTMOVE(s1, d);
1002                                 M_LMUL(s2, d);
1003                         }
1004                         emit_store(jd, iptr, iptr->dst, d);
1005                         break;
1006
1007                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1008                                       /* val.l = constant                             */
1009
1010                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1011                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1012                         if (IS_IMM32(iptr->val.l))
1013                                 M_LMUL_IMM(s1, iptr->val.l, d);
1014                         else {
1015                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
1016                                 M_INTMOVE(s1, d);
1017                                 M_LMUL(REG_ITMP2, d);
1018                         }
1019                         emit_store(jd, iptr, iptr->dst, d);
1020                         break;
1021
1022                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1023
1024                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1025                 if (src->prev->flags & INMEMORY)
1026                                 M_ILD(RAX, REG_SP, src->prev->regoff * 8);
1027                         else
1028                                 M_INTMOVE(src->prev->regoff, RAX);
1029
1030                         if (src->flags & INMEMORY)
1031                                 M_ILD(REG_ITMP3, REG_SP, src->regoff * 8);
1032                         else
1033                                 M_INTMOVE(src->regoff, REG_ITMP3);
1034
1035                         if (checknull) {
1036                                 M_ITEST(REG_ITMP3);
1037                                 M_BEQ(0);
1038                                 codegen_add_arithmeticexception_ref(cd);
1039                         }
1040
1041                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1042                         emit_jcc(cd, CC_NE, 4 + 6);
1043                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
1044                         emit_jcc(cd, CC_E, 3 + 1 + 3);                /* 6 bytes */
1045
1046                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1047                         emit_cltd(cd);
1048                         emit_idivl_reg(cd, REG_ITMP3);
1049
1050                         if (iptr->dst->flags & INMEMORY) {
1051                                 emit_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1052                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1053
1054                         } else {
1055                                 M_INTMOVE(RAX, iptr->dst->regoff);
1056
1057                                 if (iptr->dst->regoff != RDX) {
1058                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1059                                 }
1060                         }
1061                         break;
1062
1063                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1064
1065                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1066                         if (src->prev->flags & INMEMORY)
1067                                 M_ILD(RAX, REG_SP, src->prev->regoff * 8);
1068                         else
1069                                 M_INTMOVE(src->prev->regoff, RAX);
1070
1071                         if (src->flags & INMEMORY)
1072                                 M_ILD(REG_ITMP3, REG_SP, src->regoff * 8);
1073                         else
1074                                 M_INTMOVE(src->regoff, REG_ITMP3);
1075
1076                         if (checknull) {
1077                                 M_ITEST(REG_ITMP3);
1078                                 M_BEQ(0);
1079                                 codegen_add_arithmeticexception_ref(cd);
1080                         }
1081
1082                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1083
1084                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1085                         emit_jcc(cd, CC_NE, 2 + 4 + 6);
1086
1087
1088                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
1089                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
1090                         emit_jcc(cd, CC_E, 1 + 3);                    /* 6 bytes */
1091
1092                         emit_cltd(cd);
1093                         emit_idivl_reg(cd, REG_ITMP3);
1094
1095                         if (iptr->dst->flags & INMEMORY) {
1096                                 emit_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1097                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1098
1099                         } else {
1100                                 M_INTMOVE(RDX, iptr->dst->regoff);
1101
1102                                 if (iptr->dst->regoff != RDX) {
1103                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1104                                 }
1105                         }
1106                         break;
1107
1108                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1109                                       /* val.i = constant                             */
1110
1111                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1112                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1113                         M_INTMOVE(s1, REG_ITMP1);
1114                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1115                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1116                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
1117                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->val.i, REG_ITMP1);
1118                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1119                         emit_store(jd, iptr, iptr->dst, d);
1120                         break;
1121
1122                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1123                                       /* val.i = constant                             */
1124
1125                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1126                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1127                         M_INTMOVE(s1, REG_ITMP1);
1128                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1129                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1130                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
1131                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->val.i), REG_ITMP2);
1132                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
1133                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1134                         emit_store(jd, iptr, iptr->dst, d);
1135                         break;
1136
1137
1138                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1139
1140                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1141
1142                 if (src->prev->flags & INMEMORY)
1143                                 M_LLD(RAX, REG_SP, src->prev->regoff * 8);
1144                         else
1145                                 M_INTMOVE(src->prev->regoff, RAX);
1146
1147                         if (src->flags & INMEMORY)
1148                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1149                         else
1150                                 M_INTMOVE(src->regoff, REG_ITMP3);
1151
1152                         if (checknull) {
1153                                 M_TEST(REG_ITMP3);
1154                                 M_BEQ(0);
1155                                 codegen_add_arithmeticexception_ref(cd);
1156                         }
1157
1158                         /* check as described in jvm spec */
1159                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1160                         M_LCMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, RAX);
1161                         M_BNE(4 + 6);
1162                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1163                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
1164
1165                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1166                         emit_cqto(cd);
1167                         emit_idiv_reg(cd, REG_ITMP3);
1168
1169                         if (iptr->dst->flags & INMEMORY) {
1170                                 M_LST(RAX, REG_SP, iptr->dst->regoff * 8);
1171                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1172
1173                         } else {
1174                                 M_INTMOVE(RAX, iptr->dst->regoff);
1175
1176                                 if (iptr->dst->regoff != RDX) {
1177                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1178                                 }
1179                         }
1180                         break;
1181
1182                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1183
1184                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1185                         if (src->prev->flags & INMEMORY)
1186                                 M_LLD(REG_ITMP1, REG_SP, src->prev->regoff * 8);
1187                         else
1188                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1189
1190                         if (src->flags & INMEMORY)
1191                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1192                         else
1193                                 M_INTMOVE(src->regoff, REG_ITMP3);
1194
1195                         if (checknull) {
1196                                 M_ITEST(REG_ITMP3);
1197                                 M_BEQ(0);
1198                                 codegen_add_arithmeticexception_ref(cd);
1199                         }
1200
1201                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1202
1203                         /* check as described in jvm spec */
1204                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1205                         M_LCMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP1);
1206                         M_BNE(3 + 4 + 6);
1207
1208 #if 0
1209                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
1210 #endif
1211                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
1212                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1213                         M_BEQ(2 + 3);                                          /* 6 bytes */
1214
1215                         emit_cqto(cd);
1216                         emit_idiv_reg(cd, REG_ITMP3);
1217
1218                         if (iptr->dst->flags & INMEMORY) {
1219                                 M_LST(RDX, REG_SP, iptr->dst->regoff * 8);
1220                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1221
1222                         } else {
1223                                 M_INTMOVE(RDX, iptr->dst->regoff);
1224
1225                                 if (iptr->dst->regoff != RDX) {
1226                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1227                                 }
1228                         }
1229                         break;
1230
1231                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1232                                       /* val.i = constant                             */
1233
1234                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1235                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1236                         M_INTMOVE(s1, REG_ITMP1);
1237                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1238                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1239                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
1240                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->val.i, REG_ITMP1);
1241                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1242                         emit_store(jd, iptr, iptr->dst, d);
1243                         break;
1244
1245                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1246                                       /* val.l = constant                             */
1247
1248                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1249                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1250                         M_INTMOVE(s1, REG_ITMP1);
1251                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1252                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1253                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
1254                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->val.i), REG_ITMP2);
1255                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
1256                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1257                         emit_store(jd, iptr, iptr->dst, d);
1258                         break;
1259
1260                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1261
1262                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1263                         emit_ishift(cd, SHIFT_SHL, src, iptr);
1264                         break;
1265
1266                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1267                                       /* val.i = constant                             */
1268
1269                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1270                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1271                         M_INTMOVE(s1, d);
1272                         M_ISLL_IMM(iptr->val.i, d);
1273                         emit_store(jd, iptr, iptr->dst, d);
1274                         break;
1275
1276                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1277
1278                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1279                         emit_ishift(cd, SHIFT_SAR, src, iptr);
1280                         break;
1281
1282                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1283                                       /* val.i = constant                             */
1284
1285                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1286                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1287                         M_INTMOVE(s1, d);
1288                         M_ISRA_IMM(iptr->val.i, d);
1289                         emit_store(jd, iptr, iptr->dst, d);
1290                         break;
1291
1292                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1293
1294                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1295                         emit_ishift(cd, SHIFT_SHR, src, iptr);
1296                         break;
1297
1298                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1299                                       /* val.i = constant                             */
1300
1301                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1302                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1303                         M_INTMOVE(s1, d);
1304                         M_ISRL_IMM(iptr->val.i, d);
1305                         emit_store(jd, iptr, iptr->dst, d);
1306                         break;
1307
1308                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1309
1310                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1311                         emit_lshift(cd, SHIFT_SHL, src, iptr);
1312                         break;
1313
1314         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1315                                           /* val.i = constant                             */
1316
1317                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1318                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1319                         M_INTMOVE(s1, d);
1320                         M_LSLL_IMM(iptr->val.i, d);
1321                         emit_store(jd, iptr, iptr->dst, d);
1322                         break;
1323
1324                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1325
1326                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1327                         emit_lshift(cd, SHIFT_SAR, src, iptr);
1328                         break;
1329
1330                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1331                                       /* val.i = constant                             */
1332
1333                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1334                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1335                         M_INTMOVE(s1, d);
1336                         M_LSRA_IMM(iptr->val.i, d);
1337                         emit_store(jd, iptr, iptr->dst, d);
1338                         break;
1339
1340                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1341
1342                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1343                         emit_lshift(cd, SHIFT_SHR, src, iptr);
1344                         break;
1345
1346                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1347                                       /* val.l = constant                             */
1348
1349                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1350                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1351                         M_INTMOVE(s1, d);
1352                         M_LSRL_IMM(iptr->val.i, d);
1353                         emit_store(jd, iptr, iptr->dst, d);
1354                         break;
1355
1356                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1357
1358                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1359                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1360                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1361                         if (s2 == d)
1362                                 M_IAND(s1, d);
1363                         else {
1364                                 M_INTMOVE(s1, d);
1365                                 M_IAND(s2, d);
1366                         }
1367                         emit_store(jd, iptr, iptr->dst, d);
1368                         break;
1369
1370                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1371                                       /* val.i = constant                             */
1372
1373                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1374                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1375                         M_INTMOVE(s1, d);
1376                         M_IAND_IMM(iptr->val.i, d);
1377                         emit_store(jd, iptr, iptr->dst, d);
1378                         break;
1379
1380                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1381
1382                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1383                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1384                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1385                         if (s2 == d)
1386                                 M_LAND(s1, d);
1387                         else {
1388                                 M_INTMOVE(s1, d);
1389                                 M_LAND(s2, d);
1390                         }
1391                         emit_store(jd, iptr, iptr->dst, d);
1392                         break;
1393
1394                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1395                                       /* val.l = constant                             */
1396
1397                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1398                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1399                         M_INTMOVE(s1, d);
1400                         if (IS_IMM32(iptr->val.l))
1401                                 M_LAND_IMM(iptr->val.l, d);
1402                         else {
1403                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
1404                                 M_LAND(REG_ITMP2, d);
1405                         }
1406                         emit_store(jd, iptr, iptr->dst, d);
1407                         break;
1408
1409                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1410
1411                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1412                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1413                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1414                         if (s2 == d)
1415                                 M_IOR(s1, d);
1416                         else {
1417                                 M_INTMOVE(s1, d);
1418                                 M_IOR(s2, d);
1419                         }
1420                         emit_store(jd, iptr, iptr->dst, d);
1421                         break;
1422
1423                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1424                                       /* val.i = constant                             */
1425
1426                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1427                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1428                         M_INTMOVE(s1, d);
1429                         M_IOR_IMM(iptr->val.i, d);
1430                         emit_store(jd, iptr, iptr->dst, d);
1431                         break;
1432
1433                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1434
1435                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1436                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1437                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1438                         if (s2 == d)
1439                                 M_LOR(s1, d);
1440                         else {
1441                                 M_INTMOVE(s1, d);
1442                                 M_LOR(s2, d);
1443                         }
1444                         emit_store(jd, iptr, iptr->dst, d);
1445                         break;
1446
1447                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1448                                       /* val.l = constant                             */
1449
1450                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1451                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1452                         M_INTMOVE(s1, d);
1453                         if (IS_IMM32(iptr->val.l))
1454                                 M_LOR_IMM(iptr->val.l, d);
1455                         else {
1456                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
1457                                 M_LOR(REG_ITMP2, d);
1458                         }
1459                         emit_store(jd, iptr, iptr->dst, d);
1460                         break;
1461
1462                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1463
1464                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1465                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1466                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1467                         if (s2 == d)
1468                                 M_IXOR(s1, d);
1469                         else {
1470                                 M_INTMOVE(s1, d);
1471                                 M_IXOR(s2, d);
1472                         }
1473                         emit_store(jd, iptr, iptr->dst, d);
1474                         break;
1475
1476                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1477                                       /* val.i = constant                             */
1478
1479                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1480                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1481                         M_INTMOVE(s1, d);
1482                         M_IXOR_IMM(iptr->val.i, d);
1483                         emit_store(jd, iptr, iptr->dst, d);
1484                         break;
1485
1486                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1487
1488                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1489                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1490                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1491                         if (s2 == d)
1492                                 M_LXOR(s1, d);
1493                         else {
1494                                 M_INTMOVE(s1, d);
1495                                 M_LXOR(s2, d);
1496                         }
1497                         emit_store(jd, iptr, iptr->dst, d);
1498                         break;
1499
1500                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1501                                       /* val.l = constant                             */
1502
1503                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1504                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1505                         M_INTMOVE(s1, d);
1506                         if (IS_IMM32(iptr->val.l))
1507                                 M_LXOR_IMM(iptr->val.l, d);
1508                         else {
1509                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
1510                                 M_LXOR(REG_ITMP2, d);
1511                         }
1512                         emit_store(jd, iptr, iptr->dst, d);
1513                         break;
1514
1515
1516                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1517                                       /* op1 = variable, val.i = constant             */
1518
1519                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1520                         if (var->flags & INMEMORY) {
1521                                 s1 = REG_ITMP1;
1522                                 M_ILD(s1, REG_SP, var->regoff * 8);
1523                         } else
1524                                 s1 = var->regoff;
1525
1526                         /* Using inc and dec is not faster than add (tested with
1527                            sieve). */
1528
1529                         M_IADD_IMM(iptr->val.i, s1);
1530
1531                         if (var->flags & INMEMORY)
1532                                 M_IST(s1, REG_SP, var->regoff * 8);
1533                         break;
1534
1535
1536                 /* floating operations ************************************************/
1537
1538                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1539
1540                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1541                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1542                         disp = dseg_adds4(cd, 0x80000000);
1543                         M_FLTMOVE(s1, d);
1544                         emit_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1545                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1546                         emit_store(jd, iptr, iptr->dst, d);
1547                         break;
1548
1549                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1550
1551                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1552                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1553                         disp = dseg_adds8(cd, 0x8000000000000000);
1554                         M_FLTMOVE(s1, d);
1555                         emit_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1556                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1557                         emit_store(jd, iptr, iptr->dst, d);
1558                         break;
1559
1560                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1561
1562                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1563                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1564                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1565                         if (s2 == d)
1566                                 M_FADD(s1, d);
1567                         else {
1568                                 M_FLTMOVE(s1, d);
1569                                 M_FADD(s2, d);
1570                         }
1571                         emit_store(jd, iptr, iptr->dst, d);
1572                         break;
1573
1574                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1575
1576                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1577                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1578                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1579                         if (s2 == d)
1580                                 M_DADD(s1, d);
1581                         else {
1582                                 M_FLTMOVE(s1, d);
1583                                 M_DADD(s2, d);
1584                         }
1585                         emit_store(jd, iptr, iptr->dst, d);
1586                         break;
1587
1588                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1589
1590                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1591                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1592                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1593                         if (s2 == d) {
1594                                 M_FLTMOVE(s2, REG_FTMP2);
1595                                 s2 = REG_FTMP2;
1596                         }
1597                         M_FLTMOVE(s1, d);
1598                         M_FSUB(s2, d);
1599                         emit_store(jd, iptr, iptr->dst, d);
1600                         break;
1601
1602                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1603
1604                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1605                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1606                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1607                         if (s2 == d) {
1608                                 M_FLTMOVE(s2, REG_FTMP2);
1609                                 s2 = REG_FTMP2;
1610                         }
1611                         M_FLTMOVE(s1, d);
1612                         M_DSUB(s2, d);
1613                         emit_store(jd, iptr, iptr->dst, d);
1614                         break;
1615
1616                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1617
1618                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1619                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1620                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1621                         if (s2 == d)
1622                                 M_FMUL(s1, d);
1623                         else {
1624                                 M_FLTMOVE(s1, d);
1625                                 M_FMUL(s2, d);
1626                         }
1627                         emit_store(jd, iptr, iptr->dst, d);
1628                         break;
1629
1630                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1631
1632                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1633                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1634                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1635                         if (s2 == d)
1636                                 M_DMUL(s1, d);
1637                         else {
1638                                 M_FLTMOVE(s1, d);
1639                                 M_DMUL(s2, d);
1640                         }
1641                         emit_store(jd, iptr, iptr->dst, d);
1642                         break;
1643
1644                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1645
1646                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1647                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1648                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1649                         if (s2 == d) {
1650                                 M_FLTMOVE(s2, REG_FTMP2);
1651                                 s2 = REG_FTMP2;
1652                         }
1653                         M_FLTMOVE(s1, d);
1654                         M_FDIV(s2, d);
1655                         emit_store(jd, iptr, iptr->dst, d);
1656                         break;
1657
1658                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1659
1660                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1661                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1662                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1663                         if (s2 == d) {
1664                                 M_FLTMOVE(s2, REG_FTMP2);
1665                                 s2 = REG_FTMP2;
1666                         }
1667                         M_FLTMOVE(s1, d);
1668                         M_DDIV(s2, d);
1669                         emit_store(jd, iptr, iptr->dst, d);
1670                         break;
1671
1672                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1673
1674                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1675                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1676                         M_CVTIF(s1, d);
1677                         emit_store(jd, iptr, iptr->dst, d);
1678                         break;
1679
1680                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1681
1682                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1683                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1684                         M_CVTID(s1, d);
1685                         emit_store(jd, iptr, iptr->dst, d);
1686                         break;
1687
1688                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1689
1690                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1691                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1692                         M_CVTLF(s1, d);
1693                         emit_store(jd, iptr, iptr->dst, d);
1694                         break;
1695
1696                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1697
1698                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1699                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1700                         M_CVTLD(s1, d);
1701                         emit_store(jd, iptr, iptr->dst, d);
1702                         break;
1703
1704                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1705
1706                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1707                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1708                         M_CVTFI(s1, d);
1709                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1710                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1711                                 ((REG_RESULT == d) ? 0 : 3);
1712                         M_BNE(disp);
1713                         M_FLTMOVE(s1, REG_FTMP1);
1714                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1715                         M_CALL(REG_ITMP2);
1716                         M_INTMOVE(REG_RESULT, d);
1717                         emit_store(jd, iptr, iptr->dst, d);
1718                         break;
1719
1720                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1721
1722                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1723                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1724                         M_CVTDI(s1, d);
1725                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1726                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1727                                 ((REG_RESULT == d) ? 0 : 3);
1728                         M_BNE(disp);
1729                         M_FLTMOVE(s1, REG_FTMP1);
1730                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1731                         M_CALL(REG_ITMP2);
1732                         M_INTMOVE(REG_RESULT, d);
1733                         emit_store(jd, iptr, iptr->dst, d);
1734                         break;
1735
1736                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1737
1738                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1739                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1740                         M_CVTFL(s1, d);
1741                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1742                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1743                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1744                                 ((REG_RESULT == d) ? 0 : 3);
1745                         M_BNE(disp);
1746                         M_FLTMOVE(s1, REG_FTMP1);
1747                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1748                         M_CALL(REG_ITMP2);
1749                         M_INTMOVE(REG_RESULT, d);
1750                         emit_store(jd, iptr, iptr->dst, d);
1751                         break;
1752
1753                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1754
1755                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1756                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1757                         M_CVTDL(s1, d);
1758                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1759                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1760                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1761                                 ((REG_RESULT == d) ? 0 : 3);
1762                         M_BNE(disp);
1763                         M_FLTMOVE(s1, REG_FTMP1);
1764                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1765                         M_CALL(REG_ITMP2);
1766                         M_INTMOVE(REG_RESULT, d);
1767                         emit_store(jd, iptr, iptr->dst, d);
1768                         break;
1769
1770                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1771
1772                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1773                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1774                         M_CVTFD(s1, d);
1775                         emit_store(jd, iptr, iptr->dst, d);
1776                         break;
1777
1778                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1779
1780                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1781                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1782                         M_CVTDF(s1, d);
1783                         emit_store(jd, iptr, iptr->dst, d);
1784                         break;
1785
1786                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1787                                           /* == => 0, < => 1, > => -1 */
1788
1789                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1790                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1791                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1792                         M_CLR(d);
1793                         M_MOV_IMM(1, REG_ITMP1);
1794                         M_MOV_IMM(-1, REG_ITMP2);
1795                         emit_ucomiss_reg_reg(cd, s1, s2);
1796                         M_CMOVB(REG_ITMP1, d);
1797                         M_CMOVA(REG_ITMP2, d);
1798                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1799                         emit_store(jd, iptr, iptr->dst, d);
1800                         break;
1801
1802                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1803                                           /* == => 0, < => 1, > => -1 */
1804
1805                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1806                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1807                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1808                         M_CLR(d);
1809                         M_MOV_IMM(1, REG_ITMP1);
1810                         M_MOV_IMM(-1, REG_ITMP2);
1811                         emit_ucomiss_reg_reg(cd, s1, s2);
1812                         M_CMOVB(REG_ITMP1, d);
1813                         M_CMOVA(REG_ITMP2, d);
1814                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1815                         emit_store(jd, iptr, iptr->dst, d);
1816                         break;
1817
1818                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1819                                           /* == => 0, < => 1, > => -1 */
1820
1821                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1822                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1823                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1824                         M_CLR(d);
1825                         M_MOV_IMM(1, REG_ITMP1);
1826                         M_MOV_IMM(-1, REG_ITMP2);
1827                         emit_ucomisd_reg_reg(cd, s1, s2);
1828                         M_CMOVB(REG_ITMP1, d);
1829                         M_CMOVA(REG_ITMP2, d);
1830                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1831                         emit_store(jd, iptr, iptr->dst, d);
1832                         break;
1833
1834                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1835                                           /* == => 0, < => 1, > => -1 */
1836
1837                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1838                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1839                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1840                         M_CLR(d);
1841                         M_MOV_IMM(1, REG_ITMP1);
1842                         M_MOV_IMM(-1, REG_ITMP2);
1843                         emit_ucomisd_reg_reg(cd, s1, s2);
1844                         M_CMOVB(REG_ITMP1, d);
1845                         M_CMOVA(REG_ITMP2, d);
1846                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1847                         emit_store(jd, iptr, iptr->dst, d);
1848                         break;
1849
1850
1851                 /* memory operations **************************************************/
1852
1853                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1854
1855                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1856                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1857                         gen_nullptr_check(s1);
1858                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1859                         emit_store(jd, iptr, iptr->dst, d);
1860                         break;
1861
1862                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1863
1864                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1865                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1866                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1867                         if (iptr->op1 == 0) {
1868                                 gen_nullptr_check(s1);
1869                                 gen_bound_check;
1870                         }
1871                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1872                         emit_store(jd, iptr, iptr->dst, d);
1873                         break;
1874
1875                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1876
1877                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1878                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1879                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1880                         if (iptr->op1 == 0) {
1881                                 gen_nullptr_check(s1);
1882                                 gen_bound_check;
1883                         }
1884                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1885                         emit_store(jd, iptr, iptr->dst, d);
1886                         break;
1887
1888                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1889
1890                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1891                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1892                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1893                         if (iptr->op1 == 0) {
1894                                 gen_nullptr_check(s1);
1895                                 gen_bound_check;
1896                         }
1897                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1898                         emit_store(jd, iptr, iptr->dst, d);
1899                         break;
1900
1901                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1902
1903                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1904                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1905                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1906                         if (iptr->op1 == 0) {
1907                                 gen_nullptr_check(s1);
1908                                 gen_bound_check;
1909                         }
1910                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1911                         emit_store(jd, iptr, iptr->dst, d);
1912                         break;
1913
1914                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1915
1916                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1917                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1918                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1919                         if (iptr->op1 == 0) {
1920                                 gen_nullptr_check(s1);
1921                                 gen_bound_check;
1922                         }
1923                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1924                         emit_store(jd, iptr, iptr->dst, d);
1925                         break;
1926
1927                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1928
1929                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1930                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1931                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1932                         if (iptr->op1 == 0) {
1933                                 gen_nullptr_check(s1);
1934                                 gen_bound_check;
1935                         }
1936                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1937                         emit_store(jd, iptr, iptr->dst, d);
1938                         break;
1939
1940                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1941
1942                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1943                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1944                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1945                         if (iptr->op1 == 0) {
1946                                 gen_nullptr_check(s1);
1947                                 gen_bound_check;
1948                         }
1949                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1950                         emit_store(jd, iptr, iptr->dst, d);
1951                         break;
1952
1953                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1954
1955                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1956                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1957                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1958                         if (iptr->op1 == 0) {
1959                                 gen_nullptr_check(s1);
1960                                 gen_bound_check;
1961                         }
1962                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1963                         emit_store(jd, iptr, iptr->dst, d);
1964                         break;
1965
1966
1967                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1968
1969                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
1970                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
1971                         if (iptr->op1 == 0) {
1972                                 gen_nullptr_check(s1);
1973                                 gen_bound_check;
1974                         }
1975                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
1976                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1977                         break;
1978
1979                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1980
1981                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
1982                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
1983                         if (iptr->op1 == 0) {
1984                                 gen_nullptr_check(s1);
1985                                 gen_bound_check;
1986                         }
1987                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
1988                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1989                         break;
1990
1991                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1992
1993                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
1994                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
1995                         if (iptr->op1 == 0) {
1996                                 gen_nullptr_check(s1);
1997                                 gen_bound_check;
1998                         }
1999                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2000                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2001                         break;
2002
2003                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2004
2005                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2006                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2007                         if (iptr->op1 == 0) {
2008                                 gen_nullptr_check(s1);
2009                                 gen_bound_check;
2010                         }
2011                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2012                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2013                         break;
2014
2015                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2016
2017                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2018                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2019                         if (iptr->op1 == 0) {
2020                                 gen_nullptr_check(s1);
2021                                 gen_bound_check;
2022                         }
2023                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2024                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2025                         break;
2026
2027                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2028
2029                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2030                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2031                         if (iptr->op1 == 0) {
2032                                 gen_nullptr_check(s1);
2033                                 gen_bound_check;
2034                         }
2035                         s3 = emit_load_s3(jd, iptr, src, REG_FTMP3);
2036                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2037                         break;
2038
2039                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2040
2041                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2042                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2043                         if (iptr->op1 == 0) {
2044                                 gen_nullptr_check(s1);
2045                                 gen_bound_check;
2046                         }
2047                         s3 = emit_load_s3(jd, iptr, src, REG_FTMP3);
2048                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2049                         break;
2050
2051                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2052
2053                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2054                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2055                         if (iptr->op1 == 0) {
2056                                 gen_nullptr_check(s1);
2057                                 gen_bound_check;
2058                         }
2059                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2060
2061                         M_MOV(s1, rd->argintregs[0]);
2062                         M_MOV(s3, rd->argintregs[1]);
2063                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
2064                         M_CALL(REG_ITMP1);
2065                         M_TEST(REG_RESULT);
2066                         M_BEQ(0);
2067                         codegen_add_arraystoreexception_ref(cd);
2068
2069                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2070                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2071                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2072                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2073                         break;
2074
2075
2076                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2077
2078                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2079                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2080                         if (iptr->op1 == 0) {
2081                                 gen_nullptr_check(s1);
2082                                 gen_bound_check;
2083                         }
2084                         emit_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2085                         break;
2086
2087                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2088
2089                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2090                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2091                         if (iptr->op1 == 0) {
2092                                 gen_nullptr_check(s1);
2093                                 gen_bound_check;
2094                         }
2095                         emit_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2096                         break;
2097
2098                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2099
2100                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2101                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2102                         if (iptr->op1 == 0) {
2103                                 gen_nullptr_check(s1);
2104                                 gen_bound_check;
2105                         }
2106                         emit_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2107                         break;
2108
2109                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2110
2111                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2112                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2113                         if (iptr->op1 == 0) {
2114                                 gen_nullptr_check(s1);
2115                                 gen_bound_check;
2116                         }
2117                         emit_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2118                         break;
2119
2120                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2121
2122                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2123                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2124                         if (iptr->op1 == 0) {
2125                                 gen_nullptr_check(s1);
2126                                 gen_bound_check;
2127                         }
2128
2129                         if (IS_IMM32(iptr->val.l)) {
2130                                 emit_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2131                         } else {
2132                                 emit_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2133                                 emit_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2134                         }
2135                         break;
2136
2137                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2138
2139                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2140                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2141                         if (iptr->op1 == 0) {
2142                                 gen_nullptr_check(s1);
2143                                 gen_bound_check;
2144                         }
2145                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2146                         break;
2147
2148
2149                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2150                                       /* op1 = type, val.a = field address            */
2151
2152                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2153                                 disp = dseg_addaddress(cd, NULL);
2154
2155 /*                              PROFILE_CYCLE_STOP; */
2156
2157                                 codegen_addpatchref(cd, PATCHER_get_putstatic,
2158                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), disp);
2159
2160                                 if (opt_showdisassemble) {
2161                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2162                                 }
2163
2164 /*                              PROFILE_CYCLE_START; */
2165
2166                         } else {
2167                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr);
2168
2169                                 disp = dseg_addaddress(cd, &(fi->value));
2170
2171                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2172                                         PROFILE_CYCLE_STOP;
2173
2174                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2175
2176                                         if (opt_showdisassemble) {
2177                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2178                                         }
2179
2180                                         PROFILE_CYCLE_START;
2181                                 }
2182                         }
2183
2184                         /* This approach is much faster than moving the field
2185                            address inline into a register. */
2186
2187                         M_ALD(REG_ITMP1, RIP, -(((ptrint) cd->mcodeptr + 7) -
2188                                                                         (ptrint) cd->mcodebase) + disp);
2189
2190                         switch (iptr->op1) {
2191                         case TYPE_INT:
2192                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
2193                                 M_ILD(d, REG_ITMP1, 0);
2194                                 break;
2195                         case TYPE_LNG:
2196                         case TYPE_ADR:
2197                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
2198                                 M_LLD(d, REG_ITMP1, 0);
2199                                 break;
2200                         case TYPE_FLT:
2201                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2202                                 M_FLD(d, REG_ITMP1, 0);
2203                                 break;
2204                         case TYPE_DBL:
2205                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2206                                 M_DLD(d, REG_ITMP1, 0);
2207                                 break;
2208                         }
2209                         emit_store(jd, iptr, iptr->dst, d);
2210                         break;
2211
2212                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2213                                       /* op1 = type, val.a = field address            */
2214
2215                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2216                                 disp = dseg_addaddress(cd, NULL);
2217
2218 /*                              PROFILE_CYCLE_STOP; */
2219
2220                                 codegen_addpatchref(cd, PATCHER_get_putstatic,
2221                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), disp);
2222
2223                                 if (opt_showdisassemble) {
2224                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2225                                 }
2226
2227 /*                              PROFILE_CYCLE_START; */
2228
2229                         } else {
2230                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr);
2231
2232                                 disp = dseg_addaddress(cd, &(fi->value));
2233
2234                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2235                                         PROFILE_CYCLE_STOP;
2236
2237                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2238
2239                                         if (opt_showdisassemble) {
2240                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2241                                         }
2242
2243                                         PROFILE_CYCLE_START;
2244                                 }
2245                         }
2246
2247                         /* This approach is much faster than moving the field
2248                            address inline into a register. */
2249
2250                         M_ALD(REG_ITMP1, RIP, -(((ptrint) cd->mcodeptr + 7) -
2251                                                                         (ptrint) cd->mcodebase) + disp);
2252
2253                         switch (iptr->op1) {
2254                         case TYPE_INT:
2255                                 s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2256                                 M_IST(s2, REG_ITMP1, 0);
2257                                 break;
2258                         case TYPE_LNG:
2259                         case TYPE_ADR:
2260                                 s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2261                                 M_LST(s2, REG_ITMP1, 0);
2262                                 break;
2263                         case TYPE_FLT:
2264                                 s2 = emit_load_s2(jd, iptr, src, REG_FTMP1);
2265                                 M_FST(s2, REG_ITMP1, 0);
2266                                 break;
2267                         case TYPE_DBL:
2268                                 s2 = emit_load_s2(jd, iptr, src, REG_FTMP1);
2269                                 M_DST(s2, REG_ITMP1, 0);
2270                                 break;
2271                         }
2272                         break;
2273
2274                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2275                                           /* val = value (in current instruction)     */
2276                                           /* op1 = type, val.a = field address (in    */
2277                                           /* following NOP)                           */
2278
2279                         if (INSTRUCTION_IS_UNRESOLVED(iptr + 1)) {
2280                                 disp = dseg_addaddress(cd, NULL);
2281
2282 /*                              PROFILE_CYCLE_STOP; */
2283
2284                                 codegen_addpatchref(cd, PATCHER_get_putstatic,
2285                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr + 1), disp);
2286
2287                                 if (opt_showdisassemble) {
2288                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2289                                 }
2290
2291 /*                              PROFILE_CYCLE_START; */
2292
2293                         } else {
2294                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr + 1);
2295
2296                                 disp = dseg_addaddress(cd, &(fi->value));
2297
2298                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2299                                         PROFILE_CYCLE_STOP;
2300
2301                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2302
2303                                         if (opt_showdisassemble) {
2304                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2305                                         }
2306
2307                                         PROFILE_CYCLE_START;
2308                                 }
2309                         }
2310
2311                         /* This approach is much faster than moving the field
2312                            address inline into a register. */
2313
2314                         M_ALD(REG_ITMP1, RIP, -(((ptrint) cd->mcodeptr + 7) -
2315                                                                         (ptrint) cd->mcodebase) + disp);
2316
2317                         switch (iptr->op1) {
2318                         case TYPE_INT:
2319                         case TYPE_FLT:
2320                                 M_IST_IMM(iptr->val.i, REG_ITMP1, 0);
2321                                 break;
2322                         case TYPE_LNG:
2323                         case TYPE_ADR:
2324                         case TYPE_DBL:
2325                                 if (IS_IMM32(iptr->val.l))
2326                                         M_LST_IMM32(iptr->val.l, REG_ITMP1, 0);
2327                                 else {
2328                                         M_IST_IMM(iptr->val.l, REG_ITMP1, 0);
2329                                         M_IST_IMM(iptr->val.l >> 32, REG_ITMP1, 4);
2330                                 }
2331                                 break;
2332                         }
2333                         break;
2334
2335                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2336                                       /* op1 = type, val.i = field offset             */
2337
2338                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2339                         gen_nullptr_check(s1);
2340
2341                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2342 /*                              PROFILE_CYCLE_STOP; */
2343
2344                                 codegen_addpatchref(cd, PATCHER_get_putfield,
2345                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), 0);
2346
2347                                 if (opt_showdisassemble) {
2348                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2349                                 }
2350
2351 /*                              PROFILE_CYCLE_START; */
2352
2353                                 disp = 0;
2354
2355                         } else
2356                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr)->offset;
2357
2358                         switch (iptr->op1) {
2359                         case TYPE_INT:
2360                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2361                                 M_ILD32(d, s1, disp);
2362                                 break;
2363                         case TYPE_LNG:
2364                         case TYPE_ADR:
2365                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2366                                 M_LLD32(d, s1, disp);
2367                                 break;
2368                         case TYPE_FLT:
2369                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2370                                 M_FLD32(d, s1, disp);
2371                                 break;
2372                         case TYPE_DBL:
2373                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2374                                 M_DLD32(d, s1, disp);
2375                                 break;
2376                         }
2377                         emit_store(jd, iptr, iptr->dst, d);
2378                         break;
2379
2380                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2381                                       /* op1 = type, val.i = field offset             */
2382
2383                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2384                         gen_nullptr_check(s1);
2385
2386                         if (IS_INT_LNG_TYPE(iptr->op1))
2387                                 s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2388                         else
2389                                 s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
2390
2391                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2392 /*                              PROFILE_CYCLE_STOP; */
2393
2394                                 codegen_addpatchref(cd, PATCHER_get_putfield,
2395                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), 0);
2396
2397                                 if (opt_showdisassemble) {
2398                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2399                                 }
2400
2401 /*                              PROFILE_CYCLE_START; */
2402
2403                                 disp = 0;
2404
2405                         } else
2406                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr)->offset;
2407
2408                         switch (iptr->op1) {
2409                         case TYPE_INT:
2410                                 M_IST32(s2, s1, disp);
2411                                 break;
2412                         case TYPE_LNG:
2413                         case TYPE_ADR:
2414                                 M_LST32(s2, s1, disp);
2415                                 break;
2416                         case TYPE_FLT:
2417                                 M_FST32(s2, s1, disp);
2418                                 break;
2419                         case TYPE_DBL:
2420                                 M_DST32(s2, s1, disp);
2421                                 break;
2422                         }
2423                         break;
2424
2425                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2426                                           /* val = value (in current instruction)     */
2427                                           /* op1 = type, val.a = field address (in    */
2428                                           /* following NOP)                           */
2429
2430                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2431                         gen_nullptr_check(s1);
2432
2433                         if (INSTRUCTION_IS_UNRESOLVED(iptr + 1)) {
2434 /*                              PROFILE_CYCLE_STOP; */
2435
2436                                 codegen_addpatchref(cd, PATCHER_putfieldconst,
2437                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr + 1), 0);
2438
2439                                 if (opt_showdisassemble) {
2440                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2441                                 }
2442
2443 /*                              PROFILE_CYCLE_START; */
2444
2445                                 disp = 0;
2446
2447                         } else
2448                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr + 1)->offset;
2449
2450                         switch (iptr->op1) {
2451                         case TYPE_INT:
2452                         case TYPE_FLT:
2453                                 M_IST32_IMM(iptr->val.i, s1, disp);
2454                                 break;
2455                         case TYPE_LNG:
2456                         case TYPE_ADR:
2457                         case TYPE_DBL:
2458                                 M_IST32_IMM(iptr->val.l, s1, disp);
2459                                 M_IST32_IMM(iptr->val.l >> 32, s1, disp + 4);
2460                                 break;
2461                         }
2462                         break;
2463
2464
2465                 /* branch operations **************************************************/
2466
2467                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2468
2469                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2470                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2471
2472                         PROFILE_CYCLE_STOP;
2473
2474 #ifdef ENABLE_VERIFIER
2475                         if (iptr->val.a) {
2476                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2477                                                                         (unresolved_class *) iptr->val.a, 0);
2478
2479                                 if (opt_showdisassemble) {
2480                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2481                                 }
2482                         }
2483 #endif /* ENABLE_VERIFIER */
2484
2485                         M_CALL_IMM(0);                            /* passing exception pc */
2486                         M_POP(REG_ITMP2_XPC);
2487
2488                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2489                         M_JMP(REG_ITMP3);
2490                         break;
2491
2492                 case ICMD_GOTO:         /* ... ==> ...                                */
2493                                         /* op1 = target JavaVM pc                     */
2494
2495                         M_JMP_IMM(0);
2496                         codegen_addreference(cd, (basicblock *) iptr->target);
2497                         break;
2498
2499                 case ICMD_JSR:          /* ... ==> ...                                */
2500                                         /* op1 = target JavaVM pc                     */
2501
2502                         M_CALL_IMM(0);
2503                         codegen_addreference(cd, (basicblock *) iptr->target);
2504                         break;
2505
2506                 case ICMD_RET:          /* ... ==> ...                                */
2507                                         /* op1 = local variable                       */
2508
2509                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2510                         if (var->flags & INMEMORY) {
2511                                 M_ALD(REG_ITMP1, REG_SP, var->regoff * 8);
2512                                 M_JMP(REG_ITMP1);
2513                         } else
2514                                 M_JMP(var->regoff);
2515                         break;
2516
2517                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2518                                         /* op1 = target JavaVM pc                     */
2519
2520                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2521                         M_TEST(s1);
2522                         M_BEQ(0);
2523                         codegen_addreference(cd, (basicblock *) iptr->target);
2524                         break;
2525
2526                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2527                                         /* op1 = target JavaVM pc                     */
2528
2529                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2530                         M_TEST(s1);
2531                         M_BNE(0);
2532                         codegen_addreference(cd, (basicblock *) iptr->target);
2533                         break;
2534
2535                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2536                                         /* op1 = target JavaVM pc, val.i = constant   */
2537
2538                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2539                         M_ICMP_IMM(iptr->val.i, s1);
2540                         M_BEQ(0);
2541                         codegen_addreference(cd, (basicblock *) iptr->target);
2542                         break;
2543
2544                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2545                                         /* op1 = target JavaVM pc, val.i = constant   */
2546
2547                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2548                         M_ICMP_IMM(iptr->val.i, s1);
2549                         M_BLT(0);
2550                         codegen_addreference(cd, (basicblock *) iptr->target);
2551                         break;
2552
2553                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2554                                         /* op1 = target JavaVM pc, val.i = constant   */
2555
2556                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2557                         M_ICMP_IMM(iptr->val.i, s1);
2558                         M_BLE(0);
2559                         codegen_addreference(cd, (basicblock *) iptr->target);
2560                         break;
2561
2562                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2563                                         /* op1 = target JavaVM pc, val.i = constant   */
2564
2565                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2566                         M_ICMP_IMM(iptr->val.i, s1);
2567                         M_BNE(0);
2568                         codegen_addreference(cd, (basicblock *) iptr->target);
2569                         break;
2570
2571                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2572                                         /* op1 = target JavaVM pc, val.i = constant   */
2573
2574                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2575                         M_ICMP_IMM(iptr->val.i, s1);
2576                         M_BGT(0);
2577                         codegen_addreference(cd, (basicblock *) iptr->target);
2578                         break;
2579
2580                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2581                                         /* op1 = target JavaVM pc, val.i = constant   */
2582
2583                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2584                         M_ICMP_IMM(iptr->val.i, s1);
2585                         M_BGE(0);
2586                         codegen_addreference(cd, (basicblock *) iptr->target);
2587                         break;
2588
2589                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2590                                         /* op1 = target JavaVM pc, val.l = constant   */
2591
2592                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2593                         if (IS_IMM32(iptr->val.l))
2594                                 M_LCMP_IMM(iptr->val.l, s1);
2595                         else {
2596                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
2597                                 M_LCMP(REG_ITMP2, s1);
2598                         }
2599                         M_BEQ(0);
2600                         codegen_addreference(cd, (basicblock *) iptr->target);
2601                         break;
2602
2603                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2604                                         /* op1 = target JavaVM pc, val.l = constant   */
2605
2606                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2607                         if (IS_IMM32(iptr->val.l))
2608                                 M_LCMP_IMM(iptr->val.l, s1);
2609                         else {
2610                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
2611                                 M_LCMP(REG_ITMP2, s1);
2612                         }
2613                         M_BLT(0);
2614                         codegen_addreference(cd, (basicblock *) iptr->target);
2615                         break;
2616
2617                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2618                                         /* op1 = target JavaVM pc, val.l = constant   */
2619
2620                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2621                         if (IS_IMM32(iptr->val.l))
2622                                 M_LCMP_IMM(iptr->val.l, s1);
2623                         else {
2624                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
2625                                 M_LCMP(REG_ITMP2, s1);
2626                         }
2627                         M_BLE(0);
2628                         codegen_addreference(cd, (basicblock *) iptr->target);
2629                         break;
2630
2631                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2632                                         /* op1 = target JavaVM pc, val.l = constant   */
2633
2634                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2635                         if (IS_IMM32(iptr->val.l))
2636                                 M_LCMP_IMM(iptr->val.l, s1);
2637                         else {
2638                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
2639                                 M_LCMP(REG_ITMP2, s1);
2640                         }
2641                         M_BNE(0);
2642                         codegen_addreference(cd, (basicblock *) iptr->target);
2643                         break;
2644
2645                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2646                                         /* op1 = target JavaVM pc, val.l = constant   */
2647
2648                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2649                         if (IS_IMM32(iptr->val.l))
2650                                 M_LCMP_IMM(iptr->val.l, s1);
2651                         else {
2652                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
2653                                 M_LCMP(REG_ITMP2, s1);
2654                         }
2655                         M_BGT(0);
2656                         codegen_addreference(cd, (basicblock *) iptr->target);
2657                         break;
2658
2659                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2660                                         /* op1 = target JavaVM pc, val.l = constant   */
2661
2662                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2663                         if (IS_IMM32(iptr->val.l))
2664                                 M_LCMP_IMM(iptr->val.l, s1);
2665                         else {
2666                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
2667                                 M_LCMP(REG_ITMP2, s1);
2668                         }
2669                         M_BGE(0);
2670                         codegen_addreference(cd, (basicblock *) iptr->target);
2671                         break;
2672
2673                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2674                                         /* op1 = target JavaVM pc                     */
2675
2676                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2677                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2678                         M_ICMP(s2, s1);
2679                         M_BEQ(0);
2680                         codegen_addreference(cd, (basicblock *) iptr->target);
2681                         break;
2682
2683                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2684                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2685
2686                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2687                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2688                         M_LCMP(s2, s1);
2689                         M_BEQ(0);
2690                         codegen_addreference(cd, (basicblock *) iptr->target);
2691                         break;
2692
2693                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2694                                         /* op1 = target JavaVM pc                     */
2695
2696                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2697                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2698                         M_ICMP(s2, s1);
2699                         M_BNE(0);
2700                         codegen_addreference(cd, (basicblock *) iptr->target);
2701                         break;
2702
2703                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2704                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2705
2706                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2707                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2708                         M_LCMP(s2, s1);
2709                         M_BNE(0);
2710                         codegen_addreference(cd, (basicblock *) iptr->target);
2711                         break;
2712
2713                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2714                                         /* op1 = target JavaVM pc                     */
2715
2716                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2717                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2718                         M_ICMP(s2, s1);
2719                         M_BLT(0);
2720                         codegen_addreference(cd, (basicblock *) iptr->target);
2721                         break;
2722
2723                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2724                                     /* op1 = target JavaVM pc                     */
2725
2726                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2727                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2728                         M_LCMP(s2, s1);
2729                         M_BLT(0);
2730                         codegen_addreference(cd, (basicblock *) iptr->target);
2731                         break;
2732
2733                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2734                                         /* op1 = target JavaVM pc                     */
2735
2736                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2737                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2738                         M_ICMP(s2, s1);
2739                         M_BGT(0);
2740                         codegen_addreference(cd, (basicblock *) iptr->target);
2741                         break;
2742
2743                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2744                                 /* op1 = target JavaVM pc                     */
2745
2746                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2747                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2748                         M_LCMP(s2, s1);
2749                         M_BGT(0);
2750                         codegen_addreference(cd, (basicblock *) iptr->target);
2751                         break;
2752
2753                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2754                                         /* op1 = target JavaVM pc                     */
2755
2756                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2757                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2758                         M_ICMP(s2, s1);
2759                         M_BLE(0);
2760                         codegen_addreference(cd, (basicblock *) iptr->target);
2761                         break;
2762
2763                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2764                                         /* op1 = target JavaVM pc                     */
2765
2766                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2767                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2768                         M_LCMP(s2, s1);
2769                         M_BLE(0);
2770                         codegen_addreference(cd, (basicblock *) iptr->target);
2771                         break;
2772
2773                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2774                                         /* op1 = target JavaVM pc                     */
2775
2776                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2777                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2778                         M_ICMP(s2, s1);
2779                         M_BGE(0);
2780                         codegen_addreference(cd, (basicblock *) iptr->target);
2781                         break;
2782
2783                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2784                                     /* op1 = target JavaVM pc                     */
2785
2786                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2787                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2788                         M_LCMP(s2, s1);
2789                         M_BGE(0);
2790                         codegen_addreference(cd, (basicblock *) iptr->target);
2791                         break;
2792
2793                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2794
2795                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2796                         break;
2797
2798                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2799                 case ICMD_IFNE_ICONST:  /* val.i = constant                           */
2800                 case ICMD_IFLT_ICONST:
2801                 case ICMD_IFGE_ICONST:
2802                 case ICMD_IFGT_ICONST:
2803                 case ICMD_IFLE_ICONST:
2804
2805                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2806                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2807                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2808                                 if (s1 == d) {
2809                                         M_INTMOVE(s1, REG_ITMP1);
2810                                         s1 = REG_ITMP1;
2811                                 }
2812                                 if (iptr[1].val.i == 0)
2813                                         M_CLR(d);
2814                                 else
2815                                         M_IMOV_IMM(iptr[1].val.i, d);
2816                         }
2817                         if (iptr->val.i == 0)
2818                                 M_CLR(REG_ITMP2);
2819                         else
2820                                 M_IMOV_IMM(iptr->val.i, REG_ITMP2);
2821                         M_ITEST(s1);
2822
2823                         switch (iptr->opc) {
2824                         case ICMD_IFEQ_ICONST:
2825                                 M_CMOVEQ(REG_ITMP2, d);
2826                                 break;
2827                         case ICMD_IFNE_ICONST:
2828                                 M_CMOVNE(REG_ITMP2, d);
2829                                 break;
2830                         case ICMD_IFLT_ICONST:
2831                                 M_CMOVLT(REG_ITMP2, d);
2832                                 break;
2833                         case ICMD_IFGE_ICONST:
2834                                 M_CMOVGE(REG_ITMP2, d);
2835                                 break;
2836                         case ICMD_IFGT_ICONST:
2837                                 M_CMOVGT(REG_ITMP2, d);
2838                                 break;
2839                         case ICMD_IFLE_ICONST:
2840                                 M_CMOVLE(REG_ITMP2, d);
2841                                 break;
2842                         }
2843
2844                         emit_store(jd, iptr, iptr->dst, d);
2845                         break;
2846
2847
2848                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2849                 case ICMD_LRETURN:
2850
2851                         s1 = emit_load_s1(jd, iptr, src, REG_RESULT);
2852                         M_INTMOVE(s1, REG_RESULT);
2853                         goto nowperformreturn;
2854
2855                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2856
2857                         s1 = emit_load_s1(jd, iptr, src, REG_RESULT);
2858                         M_INTMOVE(s1, REG_RESULT);
2859
2860 #ifdef ENABLE_VERIFIER
2861                         if (iptr->val.a) {
2862                                 PROFILE_CYCLE_STOP;
2863
2864                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2865                                                                         (unresolved_class *) iptr->val.a, 0);
2866
2867                                 if (opt_showdisassemble) {
2868                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2869                                 }
2870
2871                                 PROFILE_CYCLE_START;
2872                         }
2873 #endif /* ENABLE_VERIFIER */
2874                         goto nowperformreturn;
2875
2876                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2877                 case ICMD_DRETURN:
2878
2879                         s1 = emit_load_s1(jd, iptr, src, REG_FRESULT);
2880                         M_FLTMOVE(s1, REG_FRESULT);
2881                         goto nowperformreturn;
2882
2883                 case ICMD_RETURN:      /* ...  ==> ...                                */
2884
2885 nowperformreturn:
2886                         {
2887                         s4 i, p;
2888
2889                         p = stackframesize;
2890
2891 #if !defined(NDEBUG)
2892                         /* generate call trace */
2893
2894                         if (opt_verbosecall) {
2895                                 emit_alu_imm_reg(cd, ALU_SUB, 2 * 8, REG_SP);
2896
2897                                 emit_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2898                                 emit_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2899
2900                                 emit_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
2901                                 emit_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2902                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2903                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2904
2905                                 M_MOV_IMM(builtin_displaymethodstop, REG_ITMP1);
2906                                 M_CALL(REG_ITMP1);
2907
2908                                 emit_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2909                                 emit_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2910
2911                                 emit_alu_imm_reg(cd, ALU_ADD, 2 * 8, REG_SP);
2912                         }
2913 #endif /* !defined(NDEBUG) */
2914
2915 #if defined(ENABLE_THREADS)
2916                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2917                                 M_ALD(rd->argintregs[0], REG_SP, rd->memuse * 8);
2918
2919                                 /* we need to save the proper return value */
2920                                 switch (iptr->opc) {
2921                                 case ICMD_IRETURN:
2922                                 case ICMD_ARETURN:
2923                                 case ICMD_LRETURN:
2924                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2925                                         break;
2926                                 case ICMD_FRETURN:
2927                                 case ICMD_DRETURN:
2928                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2929                                         break;
2930                                 }
2931
2932                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2933                                 M_CALL(REG_ITMP1);
2934
2935                                 /* and now restore the proper return value */
2936                                 switch (iptr->opc) {
2937                                 case ICMD_IRETURN:
2938                                 case ICMD_ARETURN:
2939                                 case ICMD_LRETURN:
2940                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2941                                         break;
2942                                 case ICMD_FRETURN:
2943                                 case ICMD_DRETURN:
2944                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2945                                         break;
2946                                 }
2947                         }
2948 #endif
2949
2950                         /* restore saved registers */
2951
2952                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2953                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2954                         }
2955                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2956                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2957                         }
2958
2959                         /* deallocate stack */
2960
2961                         if (stackframesize)
2962                                 M_AADD_IMM(stackframesize * 8, REG_SP);
2963
2964                         /* generate method profiling code */
2965
2966                         PROFILE_CYCLE_STOP;
2967
2968                         M_RET;
2969                         }
2970                         break;
2971
2972
2973                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2974                         {
2975                                 s4 i, l, *s4ptr;
2976                                 void **tptr;
2977
2978                                 tptr = (void **) iptr->target;
2979
2980                                 s4ptr = iptr->val.a;
2981                                 l = s4ptr[1];                          /* low     */
2982                                 i = s4ptr[2];                          /* high    */
2983
2984                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2985                                 M_INTMOVE(s1, REG_ITMP1);
2986
2987                                 if (l != 0)
2988                                         M_ISUB_IMM(l, REG_ITMP1);
2989
2990                                 i = i - l + 1;
2991
2992                 /* range check */
2993                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2994                                 M_BA(0);
2995
2996                                 codegen_addreference(cd, (basicblock *) tptr[0]);
2997
2998                                 /* build jump table top down and use address of lowest entry */
2999
3000                 /* s4ptr += 3 + i; */
3001                                 tptr += i;
3002
3003                                 while (--i >= 0) {
3004                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
3005                                         --tptr;
3006                                 }
3007
3008                                 /* length of dataseg after last dseg_addtarget is used
3009                                    by load */
3010
3011                                 M_MOV_IMM(0, REG_ITMP2);
3012                                 dseg_adddata(cd);
3013                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
3014                                 M_JMP(REG_ITMP1);
3015                         }
3016                         break;
3017
3018
3019                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
3020                         {
3021                                 s4 i, l, val, *s4ptr;
3022                                 void **tptr;
3023
3024                                 tptr = (void **) iptr->target;
3025
3026                                 s4ptr = iptr->val.a;
3027                                 l = s4ptr[0];                          /* default  */
3028                                 i = s4ptr[1];                          /* count    */
3029
3030                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
3031                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3032
3033                                 while (--i >= 0) {
3034                                         s4ptr += 2;
3035                                         ++tptr;
3036
3037                                         val = s4ptr[0];
3038                                         M_ICMP_IMM(val, s1);
3039                                         M_BEQ(0);
3040                                         codegen_addreference(cd, (basicblock *) tptr[0]);
3041                                 }
3042
3043                                 M_JMP_IMM(0);
3044
3045                                 tptr = (void **) iptr->target;
3046                                 codegen_addreference(cd, (basicblock *) tptr[0]);
3047                         }
3048                         break;
3049
3050
3051                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
3052                                         /* op1 = arg count val.a = builtintable entry */
3053
3054                         bte = iptr->val.a;
3055                         md = bte->md;
3056                         goto gen_method;
3057
3058                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
3059                                         /* op1 = arg count, val.a = method pointer    */
3060
3061                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
3062                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
3063                 case ICMD_INVOKEINTERFACE:
3064
3065                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3066                                 md = INSTRUCTION_UNRESOLVED_METHOD(iptr)->methodref->parseddesc.md;
3067                                 lm = NULL;
3068                         }
3069                         else {
3070                                 lm = INSTRUCTION_RESOLVED_METHODINFO(iptr);
3071                                 md = lm->parseddesc;
3072                         }
3073
3074 gen_method:
3075                         s3 = md->paramcount;
3076
3077                         MCODECHECK((20 * s3) + 128);
3078
3079                         /* copy arguments to registers or stack location */
3080
3081                         for (s3 = s3 - 1; s3 >= 0; s3--, src = src->prev) {
3082                                 if (src->varkind == ARGVAR)
3083                                         continue;
3084                                 if (IS_INT_LNG_TYPE(src->type)) {
3085                                         if (!md->params[s3].inmemory) {
3086                                                 s1 = rd->argintregs[md->params[s3].regoff];
3087                                                 d = emit_load_s1(jd, iptr, src, s1);
3088                                                 M_INTMOVE(d, s1);
3089                                         } else {
3090                                                 d = emit_load_s1(jd, iptr, src, REG_ITMP1);
3091                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
3092                                         }
3093
3094                                 } else {
3095                                         if (!md->params[s3].inmemory) {
3096                                                 s1 = rd->argfltregs[md->params[s3].regoff];
3097                                                 d = emit_load_s1(jd, iptr, src, s1);
3098                                                 M_FLTMOVE(d, s1);
3099                                         } else {
3100                                                 d = emit_load_s1(jd, iptr, src, REG_FTMP1);
3101
3102                                                 if (IS_2_WORD_TYPE(src->type))
3103                                                         M_DST(d, REG_SP, md->params[s3].regoff * 8);
3104                                                 else
3105                                                         M_FST(d, REG_SP, md->params[s3].regoff * 8);
3106                                         }
3107                                 }
3108                         }
3109
3110                         /* generate method profiling code */
3111
3112                         PROFILE_CYCLE_STOP;
3113
3114                         switch (iptr->opc) {
3115                         case ICMD_BUILTIN:
3116                                 a = (ptrint) bte->fp;
3117                                 d = md->returntype.type;
3118
3119                                 M_MOV_IMM(a, REG_ITMP1);
3120                                 M_CALL(REG_ITMP1);
3121
3122                                 /* if op1 == true, we need to check for an exception */
3123
3124                                 if (iptr->op1 == true) {
3125                                         M_TEST(REG_RESULT);
3126                                         M_BEQ(0);
3127                                         codegen_add_fillinstacktrace_ref(cd);
3128                                 }
3129                                 break;
3130
3131                         case ICMD_INVOKESPECIAL:
3132                                 M_TEST(rd->argintregs[0]);
3133                                 M_BEQ(0);
3134                                 codegen_add_nullpointerexception_ref(cd);
3135
3136                                 /* first argument contains pointer */
3137 /*                              gen_nullptr_check(rd->argintregs[0]); */
3138
3139                                 /* access memory for hardware nullptr */
3140 /*                              emit_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); */
3141
3142                                 /* fall through */
3143
3144                         case ICMD_INVOKESTATIC:
3145                                 if (lm == NULL) {
3146                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3147
3148                                         codegen_addpatchref(cd, PATCHER_invokestatic_special,
3149                                                                                 um, 0);
3150
3151                                         if (opt_showdisassemble) {
3152                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3153                                         }
3154
3155                                         a = 0;
3156                                         d = um->methodref->parseddesc.md->returntype.type;
3157
3158                                 } else {
3159                                         a = (ptrint) lm->stubroutine;
3160                                         d = lm->parseddesc->returntype.type;
3161                                 }
3162
3163                                 M_MOV_IMM(a, REG_ITMP2);
3164                                 M_CALL(REG_ITMP2);
3165                                 break;
3166
3167                         case ICMD_INVOKEVIRTUAL:
3168                                 gen_nullptr_check(rd->argintregs[0]);
3169
3170                                 if (lm == NULL) {
3171                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3172
3173                                         codegen_addpatchref(cd, PATCHER_invokevirtual, um, 0);
3174
3175                                         if (opt_showdisassemble) {
3176                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3177                                         }
3178
3179                                         s1 = 0;
3180                                         d = um->methodref->parseddesc.md->returntype.type;
3181
3182                                 } else {
3183                                         s1 = OFFSET(vftbl_t, table[0]) +
3184                                                 sizeof(methodptr) * lm->vftblindex;
3185                                         d = lm->parseddesc->returntype.type;
3186                                 }
3187
3188                                 M_ALD(REG_METHODPTR, rd->argintregs[0],
3189                                           OFFSET(java_objectheader, vftbl));
3190                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
3191                                 M_CALL(REG_ITMP3);
3192                                 break;
3193
3194                         case ICMD_INVOKEINTERFACE:
3195                                 gen_nullptr_check(rd->argintregs[0]);
3196
3197                                 if (lm == NULL) {
3198                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3199
3200                                         codegen_addpatchref(cd, PATCHER_invokeinterface, um, 0);
3201
3202                                         if (opt_showdisassemble) {
3203                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3204                                         }
3205
3206                                         s1 = 0;
3207                                         s2 = 0;
3208                                         d = um->methodref->parseddesc.md->returntype.type;
3209
3210                                 } else {
3211                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3212                                                 sizeof(methodptr) * lm->class->index;
3213
3214                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3215
3216                                         d = lm->parseddesc->returntype.type;
3217                                 }
3218
3219                                 M_ALD(REG_METHODPTR, rd->argintregs[0],
3220                                           OFFSET(java_objectheader, vftbl));
3221                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
3222                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
3223                                 M_CALL(REG_ITMP3);
3224                                 break;
3225                         }
3226
3227                         /* generate method profiling code */
3228
3229                         PROFILE_CYCLE_START;
3230
3231                         /* d contains return type */
3232
3233                         if (d != TYPE_VOID) {
3234                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3235                                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_RESULT);
3236                                         M_INTMOVE(REG_RESULT, s1);
3237                                         emit_store(jd, iptr, iptr->dst, s1);
3238                                 } else {
3239                                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FRESULT);
3240                                         M_FLTMOVE(REG_FRESULT, s1);
3241                                         emit_store(jd, iptr, iptr->dst, s1);
3242                                 }
3243                         }
3244                         break;
3245
3246
3247                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3248
3249                                       /* op1:   0 == array, 1 == class                */
3250                                       /* val.a: (classinfo *) superclass              */
3251
3252                         /*  superclass is an interface:
3253                          *
3254                          *  OK if ((sub == NULL) ||
3255                          *         (sub->vftbl->interfacetablelength > super->index) &&
3256                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3257                          *
3258                          *  superclass is a class:
3259                          *
3260                          *  OK if ((sub == NULL) || (0
3261                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3262                          *         super->vftbl->diffval));
3263                          */
3264
3265                         if (iptr->op1 == 1) {
3266                                 /* object type cast-check */
3267
3268                                 classinfo *super;
3269                                 vftbl_t   *supervftbl;
3270                                 s4         superindex;
3271
3272                                 super = (classinfo *) iptr->val.a;
3273
3274                                 if (super == NULL) {
3275                                         superindex = 0;
3276                                         supervftbl = NULL;
3277                                 }
3278                                 else {
3279                                         superindex = super->index;
3280                                         supervftbl = super->vftbl;
3281                                 }
3282
3283 #if defined(ENABLE_THREADS)
3284                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3285 #endif
3286                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3287
3288                                 /* calculate interface checkcast code size */
3289
3290                                 s2 = 3; /* mov_membase_reg */
3291                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3292
3293                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3294                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3295                                         3 /* test */ + 6 /* jcc */;
3296
3297                                 if (super == NULL)
3298                                         s2 += (opt_showdisassemble ? 5 : 0);
3299
3300                                 /* calculate class checkcast code size */
3301
3302                                 s3 = 3; /* mov_membase_reg */
3303                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3304                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3305
3306 #if 0
3307                                 if (s1 != REG_ITMP1) {
3308                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3309                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3310                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3311                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3312                                         a += 3;    /* sub */
3313
3314                                 } else
3315 #endif
3316                                         {
3317                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3318                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3319                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3320                                         }
3321
3322                                 s3 += 3 /* cmp */ + 6 /* jcc */;
3323
3324                                 if (super == NULL)
3325                                         s3 += (opt_showdisassemble ? 5 : 0);
3326
3327                                 /* if class is not resolved, check which code to call */
3328
3329                                 if (super == NULL) {
3330                                         M_TEST(s1);
3331                                         M_BEQ(6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3332
3333                                         codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3334                                                                                 (constant_classref *) iptr->target, 0);
3335
3336                                         if (opt_showdisassemble) {
3337                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3338                                         }
3339
3340                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
3341                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
3342                                         M_BEQ(s2 + 5);
3343                                 }
3344
3345                                 /* interface checkcast code */
3346
3347                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
3348                                         if (super != NULL) {
3349                                                 M_TEST(s1);
3350                                                 M_BEQ(s2);
3351                                         }
3352
3353                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3354
3355                                         if (super == NULL) {
3356                                                 codegen_addpatchref(cd,
3357                                                                                         PATCHER_checkcast_instanceof_interface,
3358                                                                                         (constant_classref *) iptr->target,
3359                                                                                         0);
3360
3361                                                 if (opt_showdisassemble) {
3362                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3363                                                 }
3364                                         }
3365
3366                                         emit_movl_membase32_reg(cd, REG_ITMP2,
3367                                                                                           OFFSET(vftbl_t, interfacetablelength),
3368                                                                                           REG_ITMP3);
3369                                         /* XXX TWISTI: should this be int arithmetic? */
3370                                         M_LSUB_IMM32(superindex, REG_ITMP3);
3371                                         M_TEST(REG_ITMP3);
3372                                         M_BLE(0);
3373                                         codegen_add_classcastexception_ref(cd, s1);
3374                                         emit_mov_membase32_reg(cd, REG_ITMP2,
3375                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
3376                                                                                          superindex * sizeof(methodptr*),
3377                                                                                          REG_ITMP3);
3378                                         M_TEST(REG_ITMP3);
3379                                         M_BEQ(0);
3380                                         codegen_add_classcastexception_ref(cd, s1);
3381
3382                                         if (super == NULL)
3383                                                 M_JMP_IMM(s3);
3384                                 }
3385
3386                                 /* class checkcast code */
3387
3388                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
3389                                         if (super != NULL) {
3390                                                 M_TEST(s1);
3391                                                 M_BEQ(s3);
3392                                         }
3393
3394                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3395
3396                                         if (super == NULL) {
3397                                                 codegen_addpatchref(cd, PATCHER_checkcast_class,
3398                                                                                         (constant_classref *) iptr->target,
3399                                                                                         0);
3400
3401                                                 if (opt_showdisassemble) {
3402                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3403                                                 }
3404                                         }
3405
3406                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3407 #if defined(ENABLE_THREADS)
3408                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3409 #endif
3410                                         emit_movl_membase32_reg(cd, REG_ITMP2,
3411                                                                                           OFFSET(vftbl_t, baseval),
3412                                                                                           REG_ITMP2);
3413                                         /*                                      if (s1 != REG_ITMP1) { */
3414                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3415                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
3416                                         /*                                                                                              REG_ITMP1); */
3417                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3418                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3419                                         /*                                                                                              REG_ITMP3); */
3420                                         /*  #if defined(ENABLE_THREADS) */
3421                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3422                                         /*  #endif */
3423                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
3424
3425                                         /*                                      } else { */
3426                                         emit_movl_membase32_reg(cd, REG_ITMP3,
3427                                                                                           OFFSET(vftbl_t, baseval),
3428                                                                                           REG_ITMP3);
3429                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3430                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3431                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3432                                         /*                                      } */
3433 #if defined(ENABLE_THREADS)
3434                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3435 #endif
3436                                         M_LCMP(REG_ITMP3, REG_ITMP2);
3437                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3438                                         codegen_add_classcastexception_ref(cd, s1);
3439                                 }
3440
3441                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
3442                         }
3443                         else {
3444                                 /* array type cast-check */
3445
3446                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP2);
3447                                 M_INTMOVE(s1, rd->argintregs[0]);
3448
3449                                 if (iptr->val.a == NULL) {
3450                                         codegen_addpatchref(cd, PATCHER_builtin_arraycheckcast,
3451                                                                                 (constant_classref *) iptr->target, 0);
3452
3453                                         if (opt_showdisassemble) {
3454                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3455                                         }
3456                                 }
3457
3458                                 M_MOV_IMM(iptr->val.a, rd->argintregs[1]);
3459                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
3460                                 M_CALL(REG_ITMP1);
3461
3462                                 /* s1 may have been destroyed over the function call */
3463                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP2);
3464                                 M_TEST(REG_RESULT);
3465                                 M_BEQ(0);
3466                                 codegen_add_classcastexception_ref(cd, s1);
3467
3468                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
3469                         }
3470
3471                         M_INTMOVE(s1, d);
3472                         emit_store(jd, iptr, iptr->dst, d);
3473                         break;
3474
3475                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3476
3477                                       /* op1:   0 == array, 1 == class                */
3478                                       /* val.a: (classinfo *) superclass              */
3479
3480                         /*  superclass is an interface:
3481                          *
3482                          *  return (sub != NULL) &&
3483                          *         (sub->vftbl->interfacetablelength > super->index) &&
3484                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3485                          *
3486                          *  superclass is a class:
3487                          *
3488                          *  return ((sub != NULL) && (0
3489                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3490                          *          super->vftbl->diffvall));
3491                          */
3492
3493                         {
3494                         classinfo *super;
3495                         vftbl_t   *supervftbl;
3496                         s4         superindex;
3497
3498                         super = (classinfo *) iptr->val.a;
3499
3500                         if (!super) {
3501                                 superindex = 0;
3502                                 supervftbl = NULL;
3503
3504                         } else {
3505                                 superindex = super->index;
3506                                 supervftbl = super->vftbl;
3507                         }
3508
3509 #if defined(ENABLE_THREADS)
3510             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3511 #endif
3512
3513                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3514                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
3515                         if (s1 == d) {
3516                                 M_INTMOVE(s1, REG_ITMP1);
3517                                 s1 = REG_ITMP1;
3518                         }
3519
3520                         /* calculate interface instanceof code size */
3521
3522                         s2 = 3; /* mov_membase_reg */
3523                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3524                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3525                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3526                                 3 /* test */ + 4 /* setcc */;
3527
3528                         if (!super)
3529                                 s2 += (opt_showdisassemble ? 5 : 0);
3530
3531                         /* calculate class instanceof code size */
3532
3533                         s3 = 3; /* mov_membase_reg */
3534                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3535                         s3 += 10; /* mov_imm_reg */
3536                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3537                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3538                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3539                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3540                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3541                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3542                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3543
3544                         if (!super)
3545                                 s3 += (opt_showdisassemble ? 5 : 0);
3546
3547                         emit_alu_reg_reg(cd, ALU_XOR, d, d);
3548
3549                         /* if class is not resolved, check which code to call */
3550
3551                         if (!super) {
3552                                 emit_test_reg_reg(cd, s1, s1);
3553                                 emit_jcc(cd, CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3554                                                                                          7 + 6 + s2 + 5 + s3));
3555
3556                                 codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3557                                                                         (constant_classref *) iptr->target, 0);
3558
3559                                 if (opt_showdisassemble) {
3560                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3561                                 }
3562
3563                                 emit_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3564                                 emit_alul_imm_reg(cd, ALU_AND, ACC_INTERFACE, REG_ITMP3);
3565                                 emit_jcc(cd, CC_Z, s2 + 5);
3566                         }
3567
3568                         /* interface instanceof code */
3569
3570                         if (!super || (super->flags & ACC_INTERFACE)) {
3571                                 if (super) {
3572                                         emit_test_reg_reg(cd, s1, s1);
3573                                         emit_jcc(cd, CC_Z, s2);
3574                                 }
3575
3576                                 emit_mov_membase_reg(cd, s1,
3577                                                                            OFFSET(java_objectheader, vftbl),
3578                                                                            REG_ITMP1);
3579                                 if (!super) {
3580                                         codegen_addpatchref(cd,
3581                                                                                 PATCHER_checkcast_instanceof_interface,
3582                                                                                 (constant_classref *) iptr->target, 0);
3583
3584                                         if (opt_showdisassemble) {
3585                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3586                                         }
3587                                 }
3588
3589                                 emit_movl_membase32_reg(cd, REG_ITMP1,
3590                                                                                   OFFSET(vftbl_t, interfacetablelength),
3591                                                                                   REG_ITMP3);
3592                                 emit_alu_imm32_reg(cd, ALU_SUB, superindex, REG_ITMP3);
3593                                 emit_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3594
3595                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3596
3597                                 emit_jcc(cd, CC_LE, a);
3598                                 emit_mov_membase32_reg(cd, REG_ITMP1,
3599                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3600                                                                                  superindex * sizeof(methodptr*),
3601                                                                                  REG_ITMP1);
3602                                 emit_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3603                                 emit_setcc_reg(cd, CC_NE, d);
3604
3605                                 if (!super)
3606                                         emit_jmp_imm(cd, s3);
3607                         }
3608
3609                         /* class instanceof code */
3610
3611                         if (!super || !(super->flags & ACC_INTERFACE)) {
3612                                 if (super) {
3613                                         emit_test_reg_reg(cd, s1, s1);
3614                                         emit_jcc(cd, CC_E, s3);
3615                                 }
3616
3617                                 emit_mov_membase_reg(cd, s1,
3618                                                                            OFFSET(java_objectheader, vftbl),
3619                                                                            REG_ITMP1);
3620
3621                                 if (!super) {
3622                                         codegen_addpatchref(cd, PATCHER_instanceof_class,
3623                                                                                 (constant_classref *) iptr->target, 0);
3624
3625                                         if (opt_showdisassemble) {
3626                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3627                                         }
3628                                 }
3629
3630                                 emit_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3631 #if defined(ENABLE_THREADS)
3632                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3633 #endif
3634                                 emit_movl_membase_reg(cd, REG_ITMP1,
3635                                                                                 OFFSET(vftbl_t, baseval),
3636                                                                                 REG_ITMP1);
3637                                 emit_movl_membase_reg(cd, REG_ITMP2,
3638                                                                                 OFFSET(vftbl_t, diffval),
3639                                                                                 REG_ITMP3);
3640                                 emit_movl_membase_reg(cd, REG_ITMP2,
3641                                                                                 OFFSET(vftbl_t, baseval),
3642                                                                                 REG_ITMP2);
3643 #if defined(ENABLE_THREADS)
3644                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3645 #endif
3646                                 emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
3647                                 emit_alu_reg_reg(cd, ALU_XOR, d, d); /* may be REG_ITMP2 */
3648                                 emit_alu_reg_reg(cd, ALU_CMP, REG_ITMP3, REG_ITMP1);
3649                                 emit_setcc_reg(cd, CC_BE, d);
3650                         }
3651                         emit_store(jd, iptr, iptr->dst, d);
3652                         }
3653                         break;
3654
3655                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3656                                          /* op1 = dimension, val.a = class            */
3657
3658                         /* check for negative sizes and copy sizes to stack if necessary  */
3659
3660                         MCODECHECK((10 * 4 * iptr->op1) + 5 + 10 * 8);
3661
3662                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3663                                 /* copy SAVEDVAR sizes to stack */
3664
3665                                 if (src->varkind != ARGVAR) {
3666                                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP1);
3667                                         M_LST(s2, REG_SP, s1 * 8);
3668                                 }
3669                         }
3670
3671                         /* is a patcher function set? */
3672
3673                         if (iptr->val.a == NULL) {
3674                                 codegen_addpatchref(cd, PATCHER_builtin_multianewarray,
3675                                                                         (constant_classref *) iptr->target, 0);
3676
3677                                 if (opt_showdisassemble) {
3678                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3679                                 }
3680
3681                                 a = 0;
3682
3683                         } else {
3684                                 a = (ptrint) iptr->val.a;
3685                         }
3686
3687                         /* a0 = dimension count */
3688
3689                         M_MOV_IMM(iptr->op1, rd->argintregs[0]);
3690
3691                         /* a1 = arrayvftbl */
3692
3693                         M_MOV_IMM(iptr->val.a, rd->argintregs[1]);
3694
3695                         /* a2 = pointer to dimensions = stack pointer */
3696
3697                         M_MOV(REG_SP, rd->argintregs[2]);
3698
3699                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3700                         M_CALL(REG_ITMP1);
3701
3702                         /* check for exception before result assignment */
3703
3704                         M_TEST(REG_RESULT);
3705                         M_BEQ(0);
3706                         codegen_add_fillinstacktrace_ref(cd);
3707
3708                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_RESULT);
3709                         M_INTMOVE(REG_RESULT, s1);
3710                         emit_store(jd, iptr, iptr->dst, s1);
3711                         break;
3712
3713                 default:
3714                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3715                         return false;
3716         } /* switch */
3717
3718         } /* for instruction */
3719
3720         /* copy values to interface registers */
3721
3722         src = bptr->outstack;
3723         len = bptr->outdepth;
3724         MCODECHECK(512);
3725 #if defined(ENABLE_LSRA)
3726         if (!opt_lsra)
3727 #endif
3728         while (src) {
3729                 len--;
3730                 if ((src->varkind != STACKVAR)) {
3731                         s2 = src->type;
3732                         if (IS_FLT_DBL_TYPE(s2)) {
3733                                 s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
3734                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3735                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3736
3737                                 } else {
3738                                         emit_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3739                                 }
3740
3741                         } else {
3742                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3743                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3744                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3745
3746                                 } else {
3747                                         emit_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3748                                 }
3749                         }
3750                 }
3751                 src = src->prev;
3752         }
3753
3754         /* At the end of a basic block we may have to append some nops,
3755            because the patcher stub calling code might be longer than the
3756            actual instruction. So codepatching does not change the
3757            following block unintentionally. */
3758
3759         if (cd->mcodeptr < cd->lastmcodeptr) {
3760                 while (cd->mcodeptr < cd->lastmcodeptr) {
3761                         M_NOP;
3762                 }
3763         }
3764
3765         } /* if (bptr -> flags >= BBREACHED) */
3766         } /* for basic block */
3767
3768         dseg_createlinenumbertable(cd);
3769
3770
3771         /* generate exception and patcher stubs */
3772
3773         {
3774                 exceptionref *eref;
3775                 patchref     *pref;
3776                 ptrint        mcode;
3777                 u1           *savedmcodeptr;
3778                 u1           *tmpmcodeptr;
3779
3780                 savedmcodeptr = NULL;
3781
3782                 /* generate exception stubs */
3783
3784                 for (eref = cd->exceptionrefs; eref != NULL; eref = eref->next) {
3785                         gen_resolvebranch(cd->mcodebase + eref->branchpos,
3786                                                           eref->branchpos,
3787                                                           cd->mcodeptr - cd->mcodebase);
3788
3789                         MCODECHECK(512);
3790
3791                         /* Check if the exception is an
3792                            ArrayIndexOutOfBoundsException.  If so, move index register
3793                            into a4. */
3794
3795                         if (eref->reg != -1)
3796                                 M_MOV(eref->reg, rd->argintregs[4]);
3797
3798                         /* calcuate exception address */
3799
3800                         M_MOV_IMM(0, rd->argintregs[3]);
3801                         dseg_adddata(cd);
3802                         M_AADD_IMM32(eref->branchpos - 6, rd->argintregs[3]);
3803
3804                         /* move function to call into REG_ITMP3 */
3805
3806                         M_MOV_IMM(eref->function, REG_ITMP3);
3807
3808                         if (savedmcodeptr != NULL) {
3809                                 M_JMP_IMM(savedmcodeptr - cd->mcodeptr - 5);
3810
3811                         } else {
3812                                 savedmcodeptr = cd->mcodeptr;
3813
3814                                 emit_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3815                                 M_MOV(REG_SP, rd->argintregs[1]);
3816                                 M_ALD(rd->argintregs[2], REG_SP, stackframesize * 8);
3817
3818                                 M_ASUB_IMM(2 * 8, REG_SP);
3819                                 M_AST(rd->argintregs[3], REG_SP, 0 * 8);         /* store XPC */
3820
3821                                 M_CALL(REG_ITMP3);
3822
3823                                 M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3824                                 M_AADD_IMM(2 * 8, REG_SP);
3825
3826                                 M_MOV_IMM(asm_handle_exception, REG_ITMP3);
3827                                 M_JMP(REG_ITMP3);
3828                         }
3829                 }
3830
3831
3832                 /* generate code patching stub call code */
3833
3834                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
3835                         /* check size of code segment */
3836
3837                         MCODECHECK(512);
3838
3839                         /* Get machine code which is patched back in later. A
3840                            `call rel32' is 5 bytes long (but read 8 bytes). */
3841
3842                         savedmcodeptr = cd->mcodebase + pref->branchpos;
3843                         mcode = *((ptrint *) savedmcodeptr);
3844
3845                         /* patch in `call rel32' to call the following code */
3846
3847                         tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr          */
3848                         cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position */
3849
3850                         M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
3851
3852                         cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr   */
3853
3854                         /* move pointer to java_objectheader onto stack */
3855
3856 #if defined(ENABLE_THREADS)
3857                         /* create a virtual java_objectheader */
3858
3859                         (void) dseg_addaddress(cd, NULL);                         /* flcword    */
3860                         (void) dseg_addaddress(cd, lock_get_initial_lock_word()); /* monitorPtr */
3861                         a = dseg_addaddress(cd, NULL);                            /* vftbl      */
3862
3863                         emit_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP3);
3864                         M_PUSH(REG_ITMP3);
3865 #else
3866                         M_PUSH_IMM(0);
3867 #endif
3868
3869                         /* move machine code bytes and classinfo pointer into registers */
3870
3871                         M_MOV_IMM(mcode, REG_ITMP3);
3872                         M_PUSH(REG_ITMP3);
3873                         M_MOV_IMM(pref->ref, REG_ITMP3);
3874                         M_PUSH(REG_ITMP3);
3875                         M_MOV_IMM(pref->disp, REG_ITMP3);
3876                         M_PUSH(REG_ITMP3);
3877
3878                         M_MOV_IMM(pref->patcher, REG_ITMP3);
3879                         M_PUSH(REG_ITMP3);
3880
3881                         M_MOV_IMM(asm_wrapper_patcher, REG_ITMP3);
3882                         M_JMP(REG_ITMP3);
3883                 }
3884         }
3885
3886         /* generate replacement-out stubs */
3887
3888         {
3889                 int i;
3890
3891                 replacementpoint = jd->code->rplpoints;
3892
3893                 for (i = 0; i < jd->code->rplpointcount; ++i, ++replacementpoint) {
3894                         /* check code segment size */
3895
3896                         MCODECHECK(512);
3897
3898                         /* note start of stub code */
3899
3900                         replacementpoint->outcode = (u1*) (ptrint)(cd->mcodeptr - cd->mcodebase);
3901
3902                         /* make machine code for patching */
3903
3904                         disp = (ptrint)(replacementpoint->outcode - replacementpoint->pc) - 5;
3905                         replacementpoint->mcode = 0xe9 | ((u8)disp << 8);
3906
3907                         /* push address of `rplpoint` struct */
3908
3909                         M_MOV_IMM(replacementpoint, REG_ITMP3);
3910                         M_PUSH(REG_ITMP3);
3911
3912                         /* jump to replacement function */
3913
3914                         M_MOV_IMM(asm_replacement_out, REG_ITMP3);
3915                         M_JMP(REG_ITMP3);
3916                 }
3917         }
3918
3919         codegen_finish(jd);
3920
3921         /* everything's ok */
3922
3923         return true;
3924 }
3925
3926
3927 /* createcompilerstub **********************************************************
3928
3929    Creates a stub routine which calls the compiler.
3930
3931 *******************************************************************************/
3932
3933 #define COMPILERSTUB_DATASIZE    3 * SIZEOF_VOID_P
3934 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
3935
3936 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
3937
3938
3939 u1 *createcompilerstub(methodinfo *m)
3940 {
3941         u1          *s;                     /* memory to hold the stub            */
3942         ptrint      *d;
3943         codeinfo    *code;
3944         codegendata *cd;
3945         s4           dumpsize;
3946
3947         s = CNEW(u1, COMPILERSTUB_SIZE);
3948
3949         /* set data pointer and code pointer */
3950
3951         d = (ptrint *) s;
3952         s = s + COMPILERSTUB_DATASIZE;
3953
3954         /* mark start of dump memory area */
3955
3956         dumpsize = dump_size();
3957
3958         cd = DNEW(codegendata);
3959         cd->mcodeptr = s;
3960
3961         /* Store the codeinfo pointer in the same place as in the
3962            methodheader for compiled methods. */
3963
3964         code = code_codeinfo_new(m);
3965
3966         d[0] = (ptrint) asm_call_jit_compiler;
3967         d[1] = (ptrint) m;
3968         d[2] = (ptrint) code;
3969
3970         /* code for the stub */
3971
3972         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
3973         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
3974         M_JMP(REG_ITMP3);
3975
3976 #if defined(ENABLE_STATISTICS)
3977         if (opt_stat)
3978                 count_cstub_len += COMPILERSTUB_SIZE;
3979 #endif
3980
3981         /* release dump area */
3982
3983         dump_release(dumpsize);
3984
3985         return s;
3986 }
3987
3988
3989 /* createnativestub ************************************************************
3990
3991    Creates a stub routine which calls a native method.
3992
3993 *******************************************************************************/
3994
3995 u1 *createnativestub(functionptr f, jitdata *jd, methoddesc *nmd)
3996 {
3997         methodinfo   *m;
3998         codeinfo     *code;
3999         codegendata  *cd;
4000         registerdata *rd;
4001         methoddesc   *md;
4002         s4            stackframesize;       /* size of stackframe if needed       */
4003         s4            nativeparams;
4004         s4            i, j;                 /* count variables                    */
4005         s4            t;
4006         s4            s1, s2;
4007
4008         /* get required compiler data */
4009
4010         m    = jd->m;
4011         code = jd->code;
4012         cd   = jd->cd;
4013         rd   = jd->rd;
4014
4015         /* initialize variables */
4016
4017         md = m->parseddesc;
4018         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
4019
4020         /* calculate stack frame size */
4021
4022         stackframesize =
4023                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
4024                 sizeof(localref_table) / SIZEOF_VOID_P +
4025                 INT_ARG_CNT + FLT_ARG_CNT + 1 +         /* + 1 for function address   */
4026                 nmd->memuse;
4027
4028         if (!(stackframesize & 0x1))                /* keep stack 16-byte aligned */
4029                 stackframesize++;
4030
4031         /* create method header */
4032
4033         (void) dseg_addaddress(cd, code);                      /* CodeinfoPointer */
4034         (void) dseg_adds4(cd, stackframesize * 8);             /* FrameSize       */
4035         (void) dseg_adds4(cd, 0);                              /* IsSync          */
4036         (void) dseg_adds4(cd, 0);                              /* IsLeaf          */
4037         (void) dseg_adds4(cd, 0);                              /* IntSave         */
4038         (void) dseg_adds4(cd, 0);                              /* FltSave         */
4039         (void) dseg_addlinenumbertablesize(cd);
4040         (void) dseg_adds4(cd, 0);                              /* ExTableSize     */
4041
4042         /* generate native method profiling code */
4043
4044         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
4045                 /* count frequency */
4046
4047                 M_MOV_IMM(code, REG_ITMP3);
4048                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
4049         }
4050
4051         /* generate stub code */
4052
4053         M_ASUB_IMM(stackframesize * 8, REG_SP);
4054
4055 #if !defined(NDEBUG)
4056         /* generate call trace */
4057
4058         if (opt_verbosecall) {
4059                 /* save integer and float argument registers */
4060
4061                 for (i = 0, j = 1; i < md->paramcount; i++) {
4062                         if (!md->params[i].inmemory) {
4063                                 s1 = md->params[i].regoff;
4064
4065                                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4066                                         M_LST(rd->argintregs[s1], REG_SP, j * 8);
4067                                 else
4068                                         M_DST(rd->argfltregs[s1], REG_SP, j * 8);
4069
4070                                 j++;
4071                         }
4072                 }
4073
4074                 /* show integer hex code for float arguments */
4075
4076                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++) {
4077                         /* if the paramtype is a float, we have to right shift all
4078                            following integer registers */
4079
4080                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type)) {
4081                                 for (s1 = INT_ARG_CNT - 2; s1 >= i; s1--)
4082                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
4083
4084                                 emit_movd_freg_reg(cd, rd->argfltregs[j], rd->argintregs[i]);
4085                                 j++;
4086                         }
4087                 }
4088
4089                 M_MOV_IMM(m, REG_ITMP1);
4090                 M_AST(REG_ITMP1, REG_SP, 0 * 8);
4091                 M_MOV_IMM(builtin_trace_args, REG_ITMP1);
4092                 M_CALL(REG_ITMP1);
4093
4094                 /* restore integer and float argument registers */
4095
4096                 for (i = 0, j = 1; i < md->paramcount; i++) {
4097                         if (!md->params[i].inmemory) {
4098                                 s1 = md->params[i].regoff;
4099
4100                                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4101                                         M_LLD(rd->argintregs[s1], REG_SP, j * 8);
4102                                 else
4103                                         M_DLD(rd->argfltregs[s1], REG_SP, j * 8);
4104
4105                                 j++;
4106                         }
4107                 }
4108         }
4109 #endif /* !defined(NDEBUG) */
4110
4111         /* get function address (this must happen before the stackframeinfo) */
4112
4113 #if !defined(WITH_STATIC_CLASSPATH)
4114         if (f == NULL) {
4115                 codegen_addpatchref(cd, PATCHER_resolve_native, m, 0);
4116
4117                 if (opt_showdisassemble) {
4118                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
4119                 }
4120         }
4121 #endif
4122
4123         M_MOV_IMM(f, REG_ITMP3);
4124
4125
4126         /* save integer and float argument registers */
4127
4128         for (i = 0, j = 0; i < md->paramcount; i++) {
4129                 if (!md->params[i].inmemory) {
4130                         s1 = md->params[i].regoff;
4131
4132                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4133                                 M_LST(rd->argintregs[s1], REG_SP, j * 8);
4134                         else
4135                                 M_DST(rd->argfltregs[s1], REG_SP, j * 8);
4136
4137                         j++;
4138                 }
4139         }
4140
4141         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4142
4143         /* create dynamic stack info */
4144
4145         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4146         emit_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[1]);
4147         M_ALEA(REG_SP, stackframesize * 8 + SIZEOF_VOID_P, rd->argintregs[2]);
4148         M_ALD(rd->argintregs[3], REG_SP, stackframesize * 8);
4149         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
4150         M_CALL(REG_ITMP1);
4151
4152         /* restore integer and float argument registers */
4153
4154         for (i = 0, j = 0; i < md->paramcount; i++) {
4155                 if (!md->params[i].inmemory) {
4156                         s1 = md->params[i].regoff;
4157
4158                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4159                                 M_LLD(rd->argintregs[s1], REG_SP, j * 8);
4160                         else
4161                                 M_DLD(rd->argfltregs[s1], REG_SP, j * 8);
4162
4163                         j++;
4164                 }
4165         }
4166
4167         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4168
4169
4170         /* copy or spill arguments to new locations */
4171
4172         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
4173                 t = md->paramtypes[i].type;
4174
4175                 if (IS_INT_LNG_TYPE(t)) {
4176                         if (!md->params[i].inmemory) {
4177                                 s1 = rd->argintregs[md->params[i].regoff];
4178
4179                                 if (!nmd->params[j].inmemory) {
4180                                         s2 = rd->argintregs[nmd->params[j].regoff];
4181                                         M_INTMOVE(s1, s2);
4182
4183                                 } else {
4184                                         s2 = nmd->params[j].regoff;
4185                                         M_LST(s1, REG_SP, s2 * 8);
4186                                 }
4187
4188                         } else {
4189                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4190                                 s2 = nmd->params[j].regoff;
4191                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
4192                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
4193                         }
4194
4195                 } else {
4196                         /* We only copy spilled float arguments, as the float argument    */
4197                         /* registers keep unchanged.                                      */
4198
4199                         if (md->params[i].inmemory) {
4200                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4201                                 s2 = nmd->params[j].regoff;
4202
4203                                 if (IS_2_WORD_TYPE(t)) {
4204                                         M_DLD(REG_FTMP1, REG_SP, s1 * 8);
4205                                         M_DST(REG_FTMP1, REG_SP, s2 * 8);
4206                                 } else {
4207                                         M_FLD(REG_FTMP1, REG_SP, s1 * 8);
4208                                         M_FST(REG_FTMP1, REG_SP, s2 * 8);
4209                                 }
4210                         }
4211                 }
4212         }
4213
4214         /* put class into second argument register */
4215
4216         if (m->flags & ACC_STATIC)
4217                 M_MOV_IMM(m->class, rd->argintregs[1]);
4218
4219         /* put env into first argument register */
4220
4221         M_MOV_IMM(_Jv_env, rd->argintregs[0]);
4222
4223         /* do the native function call */
4224
4225         M_CALL(REG_ITMP3);
4226
4227         /* save return value */
4228
4229         if (md->returntype.type != TYPE_VOID) {
4230                 if (IS_INT_LNG_TYPE(md->returntype.type))
4231                         M_LST(REG_RESULT, REG_SP, 0 * 8);
4232                 else
4233                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
4234         }
4235
4236 #if !defined(NDEBUG)
4237         /* generate call trace */
4238
4239         if (opt_verbosecall) {
4240                 /* just restore the value we need, don't care about the other */
4241
4242                 if (md->returntype.type != TYPE_VOID) {
4243                         if (IS_INT_LNG_TYPE(md->returntype.type))
4244                                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
4245                         else
4246                                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4247                 }
4248
4249                 M_MOV_IMM(m, rd->argintregs[0]);
4250                 M_MOV(REG_RESULT, rd->argintregs[1]);
4251                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4252                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4253
4254                 M_MOV_IMM(builtin_displaymethodstop, REG_ITMP1);
4255                 M_CALL(REG_ITMP1);
4256         }
4257 #endif /* !defined(NDEBUG) */
4258
4259         /* remove native stackframe info */
4260
4261         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4262         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
4263         M_CALL(REG_ITMP1);
4264         M_MOV(REG_RESULT, REG_ITMP3);
4265
4266         /* restore return value */
4267
4268         if (md->returntype.type != TYPE_VOID) {
4269                 if (IS_INT_LNG_TYPE(md->returntype.type))
4270                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
4271                 else
4272                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4273         }
4274
4275         /* remove stackframe */
4276
4277         M_AADD_IMM(stackframesize * 8, REG_SP);
4278
4279         /* test for exception */
4280
4281         M_TEST(REG_ITMP3);
4282         M_BNE(1);
4283         M_RET;
4284
4285         /* handle exception */
4286
4287         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
4288         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
4289         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
4290
4291         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
4292         M_JMP(REG_ITMP3);
4293
4294
4295         /* process patcher calls **************************************************/
4296
4297         {
4298                 patchref *pref;
4299                 ptrint    mcode;
4300                 u1       *savedmcodeptr;
4301                 u1       *tmpmcodeptr;
4302 #if defined(ENABLE_THREADS)
4303                 s4        disp;
4304 #endif
4305
4306                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4307                         /* Get machine code which is patched back in later. A
4308                            `call rel32' is 5 bytes long (but read 8 bytes). */
4309
4310                         savedmcodeptr = cd->mcodebase + pref->branchpos;
4311                         mcode = *((ptrint *) savedmcodeptr);
4312
4313                         /* patch in `call rel32' to call the following code               */
4314
4315                         tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr          */
4316                         cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position */
4317
4318                         M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
4319
4320                         cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr   */
4321
4322                         /* move pointer to java_objectheader onto stack */
4323
4324 #if defined(ENABLE_THREADS)
4325                         /* create a virtual java_objectheader */
4326
4327                         (void) dseg_addaddress(cd, NULL);                         /* flcword    */
4328                         (void) dseg_addaddress(cd, lock_get_initial_lock_word()); /* monitorPtr */
4329                         disp = dseg_addaddress(cd, NULL);                         /* vftbl      */
4330
4331                         emit_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP3);
4332                         M_PUSH(REG_ITMP3);
4333 #else
4334                         M_PUSH_IMM(0);
4335 #endif
4336
4337                         /* move machine code bytes and classinfo pointer into registers */
4338
4339                         M_MOV_IMM(mcode, REG_ITMP3);
4340                         M_PUSH(REG_ITMP3);
4341                         M_MOV_IMM(pref->ref, REG_ITMP3);
4342                         M_PUSH(REG_ITMP3);
4343                         M_MOV_IMM(pref->disp, REG_ITMP3);
4344                         M_PUSH(REG_ITMP3);
4345
4346                         M_MOV_IMM(pref->patcher, REG_ITMP3);
4347                         M_PUSH(REG_ITMP3);
4348
4349                         M_MOV_IMM(asm_wrapper_patcher, REG_ITMP3);
4350                         M_JMP(REG_ITMP3);
4351                 }
4352         }
4353
4354         codegen_finish(jd);
4355
4356         return jd->code->entrypoint;
4357 }
4358
4359
4360 /*
4361  * These are local overrides for various environment variables in Emacs.
4362  * Please do not remove this and leave it at the end of the file, where
4363  * Emacs will automagically detect them.
4364  * ---------------------------------------------------------------------
4365  * Local variables:
4366  * mode: c
4367  * indent-tabs-mode: t
4368  * c-basic-offset: 4
4369  * tab-width: 4
4370  * End:
4371  * vim:noexpandtab:sw=4:ts=4:
4372  */