src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31             Edwin Steiner
  32
  33    $Id: codegen.c 5173 2006-07-25 15:57:11Z twisti $
  34
  35 */
  36
  37
  38 #include "config.h"
  39
  40 #include <assert.h>
  41 #include <stdio.h>
  42
  43 #include "vm/types.h"
  44
  45 #include "md.h"
  46 #include "md-abi.h"
  47
  48 #include "vm/jit/x86_64/arch.h"
  49 #include "vm/jit/x86_64/codegen.h"
  50 #include "vm/jit/x86_64/md-emit.h"
  51
  52 #include "mm/memory.h"
  53 #include "native/jni.h"
  54 #include "native/native.h"
  55
  56 #if defined(ENABLE_THREADS)
  57 # include "threads/native/lock.h"
  58 #endif
  59
  60 #include "vm/builtin.h"
  61 #include "vm/exceptions.h"
  62 #include "vm/global.h"
  63 #include "vm/loader.h"
  64 #include "vm/options.h"
  65 #include "vm/statistics.h"
  66 #include "vm/stringlocal.h"
  67 #include "vm/vm.h"
  68 #include "vm/jit/asmpart.h"
  69 #include "vm/jit/codegen-common.h"
  70 #include "vm/jit/dseg.h"
  71 #include "vm/jit/emit.h"
  72 #include "vm/jit/jit.h"
  73 #include "vm/jit/methodheader.h"
  74 #include "vm/jit/parse.h"
  75 #include "vm/jit/patcher.h"
  76 #include "vm/jit/reg.h"
  77 #include "vm/jit/replace.h"
  78
  79 #if defined(ENABLE_LSRA)
  80 # include "vm/jit/allocator/lsra.h"
  81 #endif
  82
  83
  84 /* codegen *********************************************************************
  85
  86    Generates machine code.
  87
  88 *******************************************************************************/
  89
  90 bool codegen(jitdata *jd)
  91 {
  92         methodinfo         *m;
  93         codeinfo           *code;
  94         codegendata        *cd;
  95         registerdata       *rd;
  96         s4                  len, s1, s2, s3, d, disp;
  97         u2                  currentline;
  98         ptrint              a;
  99         s4                  stackframesize;
 100         stackptr            src;
 101         varinfo            *var;
 102         basicblock         *bptr;
 103         instruction        *iptr;
 104         exceptiontable     *ex;
 105         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 106         builtintable_entry *bte;
 107         methoddesc         *md;
 108         rplpoint           *replacementpoint;
 109
 110         /* get required compiler data */
 111
 112         m    = jd->m;
 113         code = jd->code;
 114         cd   = jd->cd;
 115         rd   = jd->rd;
 116
 117         /* prevent compiler warnings */
 118
 119         d = 0;
 120         lm = NULL;
 121         bte = NULL;
 122
 123         {
 124         s4 i, p, t, l;
 125         s4 savedregs_num;
 126
 127         savedregs_num = 0;
 128
 129         /* space to save used callee saved registers */
 130
 131         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 132         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 133
 134         stackframesize = rd->memuse + savedregs_num;
 135
 136 #if defined(ENABLE_THREADS)
 137         /* space to save argument of monitor_enter */
 138
 139         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 140                 stackframesize++;
 141 #endif
 142
 143         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 144            native code e.g. libc or jni (alignment problems with
 145            movaps). */
 146
 147         if (!jd->isleafmethod || opt_verbosecall)
 148                 stackframesize |= 0x1;
 149
 150         /* create method header */
 151
 152         (void) dseg_addaddress(cd, code);                      /* CodeinfoPointer */
 153         (void) dseg_adds4(cd, stackframesize * 8);             /* FrameSize       */
 154
 155 #if defined(ENABLE_THREADS)
 156         /* IsSync contains the offset relative to the stack pointer for the
 157            argument of monitor_exit used in the exception handler. Since the
 158            offset could be zero and give a wrong meaning of the flag it is
 159            offset by one.
 160         */
 161
 162         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 163                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);       /* IsSync          */
 164         else
 165 #endif
 166                 (void) dseg_adds4(cd, 0);                          /* IsSync          */
 167
 168         (void) dseg_adds4(cd, jd->isleafmethod);               /* IsLeaf          */
 169         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave         */
 170         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave         */
 171
 172         (void) dseg_addlinenumbertablesize(cd);
 173
 174         (void) dseg_adds4(cd, cd->exceptiontablelength);       /* ExTableSize     */
 175
 176         /* create exception table */
 177
 178         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 179                 dseg_addtarget(cd, ex->start);
 180                 dseg_addtarget(cd, ex->end);
 181                 dseg_addtarget(cd, ex->handler);
 182                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 183         }
 184
 185         /* generate method profiling code */
 186
 187         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 188                 /* count frequency */
 189
 190                 M_MOV_IMM(code, REG_ITMP3);
 191                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 192
 193                 PROFILE_CYCLE_START;
 194         }
 195
 196         /* create stack frame (if necessary) */
 197
 198         if (stackframesize)
 199                 M_ASUB_IMM(stackframesize * 8, REG_SP);
 200
 201         /* save used callee saved registers */
 202
 203         p = stackframesize;
 204         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 205                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 206         }
 207         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 208                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 209         }
 210
 211         /* take arguments out of register or stack frame */
 212
 213         md = m->parseddesc;
 214
 215         for (p = 0, l = 0; p < md->paramcount; p++) {
 216                 t = md->paramtypes[p].type;
 217                 var = &(rd->locals[l][t]);
 218                 l++;
 219                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 220                         l++;
 221                 if (var->type < 0)
 222                         continue;
 223                 s1 = md->params[p].regoff;
 224                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 225                         s2 = rd->argintregs[s1];
 226                         if (!md->params[p].inmemory) {           /* register arguments    */
 227                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 228                                         M_INTMOVE(s2, var->regoff);
 229
 230                                 } else {                             /* reg arg -> spilled    */
 231                                     M_LST(s2, REG_SP, var->regoff * 8);
 232                                 }
 233
 234                         } else {                                 /* stack arguments       */
 235                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 236                                         /* + 8 for return address */
 237                                         M_LLD(var->regoff, REG_SP, (stackframesize + s1) * 8 + 8);
 238
 239                                 } else {                             /* stack arg -> spilled  */
 240                                         var->regoff = stackframesize + s1 + 1;
 241                                 }
 242                         }
 243
 244                 } else {                                     /* floating args         */
 245                         if (!md->params[p].inmemory) {           /* register arguments    */
 246                                 s2 = rd->argfltregs[s1];
 247                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 248                                         M_FLTMOVE(s2, var->regoff);
 249
 250                                 } else {                                         /* reg arg -> spilled    */
 251                                         M_DST(s2, REG_SP, var->regoff * 8);
 252                                 }
 253
 254                         } else {                                 /* stack arguments       */
 255                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 256                                         M_DLD(var->regoff, REG_SP, (stackframesize + s1) * 8 + 8);
 257
 258                                 } else {
 259                                         var->regoff = stackframesize + s1 + 1;
 260                                 }
 261                         }
 262                 }
 263         }  /* end for */
 264
 265         /* save monitorenter argument */
 266
 267 #if defined(ENABLE_THREADS)
 268         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 269                 /* stack offset for monitor argument */
 270
 271                 s1 = rd->memuse;
 272
 273                 if (opt_verbosecall) {
 274                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 275
 276                         for (p = 0; p < INT_ARG_CNT; p++)
 277                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 278
 279                         for (p = 0; p < FLT_ARG_CNT; p++)
 280                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 281
 282                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 283                 }
 284
 285                 /* decide which monitor enter function to call */
 286
 287                 if (m->flags & ACC_STATIC) {
 288                         M_MOV_IMM(&m->class->object.header, rd->argintregs[0]);
 289                 }
 290                 else {
 291                         M_TEST(rd->argintregs[0]);
 292                         M_BEQ(0);
 293                         codegen_add_nullpointerexception_ref(cd);
 294                 }
 295
 296                 M_AST(rd->argintregs[0], REG_SP, s1 * 8);
 297                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 298                 M_CALL(REG_ITMP1);
 299
 300                 if (opt_verbosecall) {
 301                         for (p = 0; p < INT_ARG_CNT; p++)
 302                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 303
 304                         for (p = 0; p < FLT_ARG_CNT; p++)
 305                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 306
 307                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 308                 }
 309         }
 310 #endif
 311
 312 #if !defined(NDEBUG)
 313         /* Copy argument registers to stack and call trace function with
 314            pointer to arguments on stack. */
 315
 316         if (opt_verbosecall) {
 317                 M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 318
 319                 /* save integer argument registers */
 320
 321                 for (p = 0; p < INT_ARG_CNT; p++)
 322                         M_LST(rd->argintregs[p], REG_SP, (1 + p) * 8);
 323
 324                 /* save float argument registers */
 325
 326                 for (p = 0; p < FLT_ARG_CNT; p++)
 327                         M_DST(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 328
 329                 /* save temporary registers for leaf methods */
 330
 331                 if (jd->isleafmethod) {
 332                         for (p = 0; p < INT_TMP_CNT; p++)
 333                                 M_LST(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 334
 335                         for (p = 0; p < FLT_TMP_CNT; p++)
 336                                 M_DST(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 337                 }
 338
 339                 /* show integer hex code for float arguments */
 340
 341                 for (p = 0, l = 0; p < md->paramcount && p < INT_ARG_CNT; p++) {
 342                         /* If the paramtype is a float, we have to right shift all
 343                            following integer registers. */
 344
 345                         if (IS_FLT_DBL_TYPE(md->paramtypes[p].type)) {
 346                                 for (s1 = INT_ARG_CNT - 2; s1 >= p; s1--)
 347                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
 348
 349                                 emit_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 350                                 l++;
 351                         }
 352                 }
 353
 354                 M_MOV_IMM(m, REG_ITMP2);
 355                 M_AST(REG_ITMP2, REG_SP, 0 * 8);
 356                 M_MOV_IMM(builtin_trace_args, REG_ITMP1);
 357                 M_CALL(REG_ITMP1);
 358
 359                 /* restore integer argument registers */
 360
 361                 for (p = 0; p < INT_ARG_CNT; p++)
 362                         M_LLD(rd->argintregs[p], REG_SP, (1 + p) * 8);
 363
 364                 /* restore float argument registers */
 365
 366                 for (p = 0; p < FLT_ARG_CNT; p++)
 367                         M_DLD(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 368
 369                 /* restore temporary registers for leaf methods */
 370
 371                 if (jd->isleafmethod) {
 372                         for (p = 0; p < INT_TMP_CNT; p++)
 373                                 M_LLD(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 374
 375                         for (p = 0; p < FLT_TMP_CNT; p++)
 376                                 M_DLD(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 377                 }
 378
 379                 M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 380         }
 381 #endif /* !defined(NDEBUG) */
 382
 383         }
 384
 385         /* end of header generation */
 386
 387         replacementpoint = jd->code->rplpoints;
 388
 389         /* walk through all basic blocks */
 390
 391         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 392
 393                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 394
 395                 if (bptr->flags >= BBREACHED) {
 396
 397                         /* branch resolving */
 398
 399                         branchref *bref;
 400                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 401                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 402                                                                   bref->branchpos,
 403                                                                   bptr->mpc);
 404                         }
 405
 406                 /* handle replacement points */
 407
 408                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 409                         replacementpoint->pc = (u1*)(ptrint)bptr->mpc; /* will be resolved later */
 410
 411                         replacementpoint++;
 412
 413                         assert(cd->lastmcodeptr <= cd->mcodeptr);
 414                         cd->lastmcodeptr = cd->mcodeptr + 5; /* 5 byte jmp patch */
 415                 }
 416
 417                 /* copy interface registers to their destination */
 418
 419                 src = bptr->instack;
 420                 len = bptr->indepth;
 421                 MCODECHECK(512);
 422
 423 #if 0
 424                 /* generate basicblock profiling code */
 425
 426                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 427                         /* count frequency */
 428
 429                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 430                         M_IINC_MEMBASE(REG_ITMP3, bptr->debug_nr * 4);
 431
 432                         /* if this is an exception handler, start profiling again */
 433
 434                         if (bptr->type == BBTYPE_EXH)
 435                                 PROFILE_CYCLE_START;
 436                 }
 437 #endif
 438
 439 #if defined(ENABLE_LSRA)
 440                 if (opt_lsra) {
 441                         while (src != NULL) {
 442                                 len--;
 443                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 444                                         if (bptr->type == BBTYPE_SBR) {
 445 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 446                                                 if (!(src->flags & INMEMORY))
 447                                                         d = src->regoff;
 448                                                 else
 449                                                         d = REG_ITMP1;
 450                                                 M_POP(d);
 451                                                 emit_store(jd, NULL, src, d);
 452
 453                                         } else if (bptr->type == BBTYPE_EXH) {
 454 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 455                                                 if (!(src->flags & INMEMORY))
 456                                                         d= src->regoff;
 457                                                 else
 458                                                         d=REG_ITMP1;
 459                                                 M_INTMOVE(REG_ITMP1, d);
 460                                                 emit_store(jd, NULL, src, d);
 461                                         }
 462                                 }
 463                                 src = src->prev;
 464                         }
 465
 466                 } else {
 467 #endif
 468
 469                 while (src != NULL) {
 470                         len--;
 471                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 472                                 if (bptr->type == BBTYPE_SBR) {
 473                                         d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 474                                         M_POP(d);
 475                                         emit_store(jd, NULL, src, d);
 476
 477                                 } else if (bptr->type == BBTYPE_EXH) {
 478                                         d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 479                                         M_INTMOVE(REG_ITMP1, d);
 480                                         emit_store(jd, NULL, src, d);
 481                                 }
 482
 483                         } else {
 484                                 d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 485                                 if ((src->varkind != STACKVAR)) {
 486                                         s2 = src->type;
 487                                         if (IS_FLT_DBL_TYPE(s2)) {
 488                                                 s1 = rd->interfaces[len][s2].regoff;
 489
 490                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 491                                                         M_FLTMOVE(s1, d);
 492                                                 else
 493                                                         M_DLD(d, REG_SP, s1 * 8);
 494
 495                                                 emit_store(jd, NULL, src, d);
 496
 497                                         } else {
 498                                                 s1 = rd->interfaces[len][s2].regoff;
 499
 500                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 501                                                         M_INTMOVE(s1, d);
 502                                                 else
 503                                                         M_LLD(d, REG_SP, s1 * 8);
 504
 505                                                 emit_store(jd, NULL, src, d);
 506                                         }
 507                                 }
 508                         }
 509                         src = src->prev;
 510                 }
 511 #if defined(ENABLE_LSRA)
 512                 }
 513 #endif
 514                 /* walk through all instructions */
 515
 516                 src = bptr->instack;
 517                 len = bptr->icount;
 518                 currentline = 0;
 519
 520                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 521                         if (iptr->line != currentline) {
 522                                 dseg_addlinenumber(cd, iptr->line);
 523                                 currentline = iptr->line;
 524                         }
 525
 526                         MCODECHECK(1024);                         /* 1KB should be enough */
 527
 528                         switch (iptr->opc) {
 529                         case ICMD_INLINE_START: /* internal ICMDs                         */
 530                         case ICMD_INLINE_END:
 531                                 break;
 532
 533                         case ICMD_NOP:    /* ...  ==> ...                                 */
 534                                 break;
 535
 536                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 537
 538                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 539                                 M_TEST(s1);
 540                                 M_BEQ(0);
 541                                 codegen_add_nullpointerexception_ref(cd);
 542                                 break;
 543
 544                 /* constant operations ************************************************/
 545
 546                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 547                                       /* op1 = 0, val.i = constant                    */
 548
 549                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 550                         ICONST(d, iptr->val.i);
 551                         emit_store(jd, iptr, iptr->dst, d);
 552                         break;
 553
 554                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 555                                       /* op1 = 0, val.l = constant                    */
 556
 557                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 558                         LCONST(d, iptr->val.l);
 559                         emit_store(jd, iptr, iptr->dst, d);
 560                         break;
 561
 562                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 563                                       /* op1 = 0, val.f = constant                    */
 564
 565                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 566                         disp = dseg_addfloat(cd, iptr->val.f);
 567                         emit_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + disp, d);
 568                         emit_store(jd, iptr, iptr->dst, d);
 569                         break;
 570
 571                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 572                                       /* op1 = 0, val.d = constant                    */
 573
 574                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 575                         disp = dseg_adddouble(cd, iptr->val.d);
 576                         emit_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, d);
 577                         emit_store(jd, iptr, iptr->dst, d);
 578                         break;
 579
 580                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 581                                       /* op1 = 0, val.a = constant                    */
 582
 583                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 584
 585                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 586 /*                              PROFILE_CYCLE_STOP; */
 587
 588                                 codegen_addpatchref(cd, PATCHER_aconst,
 589                                                                         ICMD_ACONST_UNRESOLVED_CLASSREF(iptr), 0);
 590
 591                                 if (opt_showdisassemble) {
 592                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
 593                                 }
 594
 595 /*                              PROFILE_CYCLE_START; */
 596
 597                                 M_MOV_IMM(NULL, d);
 598
 599                         } else {
 600                                 if (iptr->val.a == 0)
 601                                         M_CLR(d);
 602                                 else
 603                                         M_MOV_IMM(iptr->val.a, d);
 604                         }
 605                         emit_store(jd, iptr, iptr->dst, d);
 606                         break;
 607
 608
 609                 /* load/store operations **********************************************/
 610
 611                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 612                                       /* op1 = local variable                         */
 613
 614                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 615                         if ((iptr->dst->varkind == LOCALVAR) &&
 616                                 (iptr->dst->varnum == iptr->op1))
 617                                 break;
 618                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 619                         if (var->flags & INMEMORY)
 620                                 M_ILD(d, REG_SP, var->regoff * 8);
 621                         else
 622                                 M_INTMOVE(var->regoff, d);
 623                         emit_store(jd, iptr, iptr->dst, d);
 624                         break;
 625
 626                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 627                 case ICMD_ALOAD:      /* op1 = local variable                         */
 628
 629                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 630                         if ((iptr->dst->varkind == LOCALVAR) &&
 631                                 (iptr->dst->varnum == iptr->op1))
 632                                 break;
 633                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 634                         if (var->flags & INMEMORY)
 635                                 M_LLD(d, REG_SP, var->regoff * 8);
 636                         else
 637                                 M_INTMOVE(var->regoff, d);
 638                         emit_store(jd, iptr, iptr->dst, d);
 639                         break;
 640
 641                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 642                                       /* op1 = local variable                         */
 643
 644                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 645                         if ((iptr->dst->varkind == LOCALVAR) &&
 646                             (iptr->dst->varnum == iptr->op1))
 647                                 break;
 648                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 649                         if (var->flags & INMEMORY)
 650                                 M_FLD(d, REG_SP, var->regoff * 8);
 651                         else
 652                                 M_FLTMOVE(var->regoff, d);
 653                         emit_store(jd, iptr, iptr->dst, d);
 654                         break;
 655
 656                 case ICMD_DLOAD:      /* ...  ==> ..., content of local variable      */
 657                                       /* op1 = local variable                         */
 658
 659                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 660                         if ((iptr->dst->varkind == LOCALVAR) &&
 661                             (iptr->dst->varnum == iptr->op1))
 662                                 break;
 663                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 664                         if (var->flags & INMEMORY)
 665                                 M_DLD(d, REG_SP, var->regoff * 8);
 666                         else
 667                                 M_FLTMOVE(var->regoff, d);
 668                         emit_store(jd, iptr, iptr->dst, d);
 669                         break;
 670
 671                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 672                                       /* op1 = local variable                         */
 673
 674                         if ((src->varkind == LOCALVAR) && (src->varnum == iptr->op1))
 675                                 break;
 676                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 677                         if (var->flags & INMEMORY) {
 678                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 679                                 M_IST(s1, REG_SP, var->regoff * 8);
 680                         } else {
 681                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 682                                 M_INTMOVE(s1, var->regoff);
 683                         }
 684                         break;
 685
 686                 case ICMD_LSTORE:     /* ..., value  ==> ...                          */
 687                 case ICMD_ASTORE:     /* op1 = local variable                         */
 688
 689                         if ((src->varkind == LOCALVAR) && (src->varnum == iptr->op1))
 690                                 break;
 691                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 692                         if (var->flags & INMEMORY) {
 693                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 694                                 M_LST(s1, REG_SP, var->regoff * 8);
 695                         } else {
 696                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 697                                 M_INTMOVE(s1, var->regoff);
 698                         }
 699                         break;
 700
 701                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 702                                       /* op1 = local variable                         */
 703
 704                         if ((src->varkind == LOCALVAR) &&
 705                             (src->varnum == iptr->op1)) {
 706                                 break;
 707                         }
 708                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 709                         if (var->flags & INMEMORY) {
 710                                 s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
 711                                 M_FST(s1, REG_SP, var->regoff * 8);
 712                         } else {
 713                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 714                                 M_FLTMOVE(s1, var->regoff);
 715                         }
 716                         break;
 717
 718                 case ICMD_DSTORE:     /* ..., value  ==> ...                          */
 719                                       /* op1 = local variable                         */
 720
 721                         if ((src->varkind == LOCALVAR) &&
 722                             (src->varnum == iptr->op1)) {
 723                                 break;
 724                         }
 725                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 726                         if (var->flags & INMEMORY) {
 727                                 s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
 728                                 M_DST(s1, REG_SP, var->regoff * 8);
 729                         } else {
 730                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 731                                 M_FLTMOVE(s1, var->regoff);
 732                         }
 733                         break;
 734
 735
 736                 /* pop/dup/swap operations ********************************************/
 737
 738                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 739
 740                 case ICMD_POP:        /* ..., value  ==> ...                          */
 741                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 742                         break;
 743
 744                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 745                         M_COPY(src, iptr->dst);
 746                         break;
 747
 748                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 749
 750                         M_COPY(src,       iptr->dst);
 751                         M_COPY(src->prev, iptr->dst->prev);
 752                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 753                         break;
 754
 755                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 756
 757                         M_COPY(src,             iptr->dst);
 758                         M_COPY(src->prev,       iptr->dst->prev);
 759                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 760                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 761                         break;
 762
 763                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 764
 765                         M_COPY(src,       iptr->dst);
 766                         M_COPY(src->prev, iptr->dst->prev);
 767                         break;
 768
 769                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 770
 771                         M_COPY(src,             iptr->dst);
 772                         M_COPY(src->prev,       iptr->dst->prev);
 773                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 774                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 775                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 776                         break;
 777
 778                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 779
 780                         M_COPY(src,                   iptr->dst);
 781                         M_COPY(src->prev,             iptr->dst->prev);
 782                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 783                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 784                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 785                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 786                         break;
 787
 788                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 789
 790                         M_COPY(src,       iptr->dst->prev);
 791                         M_COPY(src->prev, iptr->dst);
 792                         break;
 793
 794
 795                 /* integer operations *************************************************/
 796
 797                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 798
 799                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 800                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 801                         M_INTMOVE(s1, d);
 802                         M_INEG(d);
 803                         emit_store(jd, iptr, iptr->dst, d);
 804                         break;
 805
 806                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 807
 808                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 809                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 810                         M_INTMOVE(s1, d);
 811                         M_LNEG(d);
 812                         emit_store(jd, iptr, iptr->dst, d);
 813                         break;
 814
 815                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 816
 817                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 818                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 819                         M_ISEXT(s1, d);
 820                         emit_store(jd, iptr, iptr->dst, d);
 821                         break;
 822
 823                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 824
 825                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 826                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 827                         M_IMOV(s1, d);
 828                         emit_store(jd, iptr, iptr->dst, d);
 829                         break;
 830
 831                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 832
 833                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 834                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 835                         M_BSEXT(s1, d);
 836                         emit_store(jd, iptr, iptr->dst, d);
 837                         break;
 838
 839                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 840
 841                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 842                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 843                         M_CZEXT(s1, d);
 844                         emit_store(jd, iptr, iptr->dst, d);
 845                         break;
 846
 847                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 848
 849                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 850                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 851                         M_SSEXT(s1, d);
 852                         emit_store(jd, iptr, iptr->dst, d);
 853                         break;
 854
 855
 856                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 857
 858                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 859                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 860                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 861                         if (s2 == d)
 862                                 M_IADD(s1, d);
 863                         else {
 864                                 M_INTMOVE(s1, d);
 865                                 M_IADD(s2, d);
 866                         }
 867                         emit_store(jd, iptr, iptr->dst, d);
 868                         break;
 869
 870                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 871                                       /* val.i = constant                             */
 872
 873                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 874                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 875                         M_INTMOVE(s1, d);
 876                         M_IADD_IMM(iptr->val.i, d);
 877                         emit_store(jd, iptr, iptr->dst, d);
 878                         break;
 879
 880                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 881
 882                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 883                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 884                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 885                         if (s2 == d)
 886                                 M_LADD(s1, d);
 887                         else {
 888                                 M_INTMOVE(s1, d);
 889                                 M_LADD(s2, d);
 890                         }
 891                         emit_store(jd, iptr, iptr->dst, d);
 892                         break;
 893
 894                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 895                                       /* val.l = constant                             */
 896
 897                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 898                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 899                         M_INTMOVE(s1, d);
 900                         if (IS_IMM32(iptr->val.l))
 901                                 M_LADD_IMM(iptr->val.l, d);
 902                         else {
 903                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
 904                                 M_LADD(REG_ITMP2, d);
 905                         }
 906                         emit_store(jd, iptr, iptr->dst, d);
 907                         break;
 908
 909                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 910
 911                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 912                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 913                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 914                         if (s2 == d) {
 915                                 M_INTMOVE(s1, REG_ITMP1);
 916                                 M_ISUB(s2, REG_ITMP1);
 917                                 M_INTMOVE(REG_ITMP1, d);
 918                         } else {
 919                                 M_INTMOVE(s1, d);
 920                                 M_ISUB(s2, d);
 921                         }
 922                         emit_store(jd, iptr, iptr->dst, d);
 923                         break;
 924
 925                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 926                                       /* val.i = constant                             */
 927
 928                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 929                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 930                         M_INTMOVE(s1, d);
 931                         M_ISUB_IMM(iptr->val.i, d);
 932                         emit_store(jd, iptr, iptr->dst, d);
 933                         break;
 934
 935                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 936
 937                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 938                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 939                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 940                         if (s2 == d) {
 941                                 M_INTMOVE(s1, REG_ITMP1);
 942                                 M_LSUB(s2, REG_ITMP1);
 943                                 M_INTMOVE(REG_ITMP1, d);
 944                         } else {
 945                                 M_INTMOVE(s1, d);
 946                                 M_LSUB(s2, d);
 947                         }
 948                         emit_store(jd, iptr, iptr->dst, d);
 949                         break;
 950
 951                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 952                                       /* val.l = constant                             */
 953
 954                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 955                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 956                         M_INTMOVE(s1, d);
 957                         if (IS_IMM32(iptr->val.l))
 958                                 M_LSUB_IMM(iptr->val.l, d);
 959                         else {
 960                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
 961                                 M_LSUB(REG_ITMP2, d);
 962                         }
 963                         emit_store(jd, iptr, iptr->dst, d);
 964                         break;
 965
 966                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 967
 968                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 969                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 970                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 971                         if (s2 == d)
 972                                 M_IMUL(s1, d);
 973                         else {
 974                                 M_INTMOVE(s1, d);
 975                                 M_IMUL(s2, d);
 976                         }
 977                         emit_store(jd, iptr, iptr->dst, d);
 978                         break;
 979
 980                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 981                                       /* val.i = constant                             */
 982
 983                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 984                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 985                         if (iptr->val.i == 2) {
 986                                 M_INTMOVE(s1, d);
 987                                 M_ISLL_IMM(1, d);
 988                         } else
 989                                 M_IMUL_IMM(s1, iptr->val.i, d);
 990                         emit_store(jd, iptr, iptr->dst, d);
 991                         break;
 992
 993                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 994
 995                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 996                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 997                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 998                         if (s2 == d)
 999                                 M_LMUL(s1, d);
1000                         else {
1001                                 M_INTMOVE(s1, d);
1002                                 M_LMUL(s2, d);
1003                         }
1004                         emit_store(jd, iptr, iptr->dst, d);
1005                         break;
1006
1007                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1008                                       /* val.l = constant                             */
1009
1010                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1011                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1012                         if (IS_IMM32(iptr->val.l))
1013                                 M_LMUL_IMM(s1, iptr->val.l, d);
1014                         else {
1015                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
1016                                 M_INTMOVE(s1, d);
1017                                 M_LMUL(REG_ITMP2, d);
1018                         }
1019                         emit_store(jd, iptr, iptr->dst, d);
1020                         break;
1021
1022                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1023
1024                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1025                 if (src->prev->flags & INMEMORY)
1026                                 M_ILD(RAX, REG_SP, src->prev->regoff * 8);
1027                         else
1028                                 M_INTMOVE(src->prev->regoff, RAX);
1029
1030                         if (src->flags & INMEMORY)
1031                                 M_ILD(REG_ITMP3, REG_SP, src->regoff * 8);
1032                         else
1033                                 M_INTMOVE(src->regoff, REG_ITMP3);
1034
1035                         if (checknull) {
1036                                 M_ITEST(REG_ITMP3);
1037                                 M_BEQ(0);
1038                                 codegen_add_arithmeticexception_ref(cd);
1039                         }
1040
1041                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1042                         emit_jcc(cd, CC_NE, 4 + 6);
1043                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
1044                         emit_jcc(cd, CC_E, 3 + 1 + 3);                /* 6 bytes */
1045
1046                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1047                         emit_cltd(cd);
1048                         emit_idivl_reg(cd, REG_ITMP3);
1049
1050                         if (iptr->dst->flags & INMEMORY) {
1051                                 emit_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1052                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1053
1054                         } else {
1055                                 M_INTMOVE(RAX, iptr->dst->regoff);
1056
1057                                 if (iptr->dst->regoff != RDX) {
1058                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1059                                 }
1060                         }
1061                         break;
1062
1063                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1064
1065                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1066                         if (src->prev->flags & INMEMORY)
1067                                 M_ILD(RAX, REG_SP, src->prev->regoff * 8);
1068                         else
1069                                 M_INTMOVE(src->prev->regoff, RAX);
1070
1071                         if (src->flags & INMEMORY)
1072                                 M_ILD(REG_ITMP3, REG_SP, src->regoff * 8);
1073                         else
1074                                 M_INTMOVE(src->regoff, REG_ITMP3);
1075
1076                         if (checknull) {
1077                                 M_ITEST(REG_ITMP3);
1078                                 M_BEQ(0);
1079                                 codegen_add_arithmeticexception_ref(cd);
1080                         }
1081
1082                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1083
1084                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1085                         emit_jcc(cd, CC_NE, 2 + 4 + 6);
1086
1087
1088                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
1089                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
1090                         emit_jcc(cd, CC_E, 1 + 3);                    /* 6 bytes */
1091
1092                         emit_cltd(cd);
1093                         emit_idivl_reg(cd, REG_ITMP3);
1094
1095                         if (iptr->dst->flags & INMEMORY) {
1096                                 emit_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1097                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1098
1099                         } else {
1100                                 M_INTMOVE(RDX, iptr->dst->regoff);
1101
1102                                 if (iptr->dst->regoff != RDX) {
1103                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1104                                 }
1105                         }
1106                         break;
1107
1108                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1109                                       /* val.i = constant                             */
1110
1111                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1112                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1113                         M_INTMOVE(s1, REG_ITMP1);
1114                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1115                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1116                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
1117                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->val.i, REG_ITMP1);
1118                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1119                         emit_store(jd, iptr, iptr->dst, d);
1120                         break;
1121
1122                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1123                                       /* val.i = constant                             */
1124
1125                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1126                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1127                         M_INTMOVE(s1, REG_ITMP1);
1128                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1129                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1130                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
1131                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->val.i), REG_ITMP2);
1132                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
1133                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1134                         emit_store(jd, iptr, iptr->dst, d);
1135                         break;
1136
1137
1138                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1139
1140                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1141
1142                 if (src->prev->flags & INMEMORY)
1143                                 M_LLD(RAX, REG_SP, src->prev->regoff * 8);
1144                         else
1145                                 M_INTMOVE(src->prev->regoff, RAX);
1146
1147                         if (src->flags & INMEMORY)
1148                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1149                         else
1150                                 M_INTMOVE(src->regoff, REG_ITMP3);
1151
1152                         if (checknull) {
1153                                 M_TEST(REG_ITMP3);
1154                                 M_BEQ(0);
1155                                 codegen_add_arithmeticexception_ref(cd);
1156                         }
1157
1158                         /* check as described in jvm spec */
1159                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1160                         M_LCMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, RAX);
1161                         M_BNE(4 + 6);
1162                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1163                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
1164
1165                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1166                         emit_cqto(cd);
1167                         emit_idiv_reg(cd, REG_ITMP3);
1168
1169                         if (iptr->dst->flags & INMEMORY) {
1170                                 M_LST(RAX, REG_SP, iptr->dst->regoff * 8);
1171                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1172
1173                         } else {
1174                                 M_INTMOVE(RAX, iptr->dst->regoff);
1175
1176                                 if (iptr->dst->regoff != RDX) {
1177                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1178                                 }
1179                         }
1180                         break;
1181
1182                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1183
1184                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1185                         if (src->prev->flags & INMEMORY)
1186                                 M_LLD(REG_ITMP1, REG_SP, src->prev->regoff * 8);
1187                         else
1188                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1189
1190                         if (src->flags & INMEMORY)
1191                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1192                         else
1193                                 M_INTMOVE(src->regoff, REG_ITMP3);
1194
1195                         if (checknull) {
1196                                 M_ITEST(REG_ITMP3);
1197                                 M_BEQ(0);
1198                                 codegen_add_arithmeticexception_ref(cd);
1199                         }
1200
1201                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1202
1203                         /* check as described in jvm spec */
1204                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1205                         M_LCMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP1);
1206                         M_BNE(3 + 4 + 6);
1207
1208 #if 0
1209                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
1210 #endif
1211                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
1212                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1213                         M_BEQ(2 + 3);                                          /* 6 bytes */
1214
1215                         emit_cqto(cd);
1216                         emit_idiv_reg(cd, REG_ITMP3);
1217
1218                         if (iptr->dst->flags & INMEMORY) {
1219                                 M_LST(RDX, REG_SP, iptr->dst->regoff * 8);
1220                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1221
1222                         } else {
1223                                 M_INTMOVE(RDX, iptr->dst->regoff);
1224
1225                                 if (iptr->dst->regoff != RDX) {
1226                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1227                                 }
1228                         }
1229                         break;
1230
1231                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1232                                       /* val.i = constant                             */
1233
1234                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1235                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1236                         M_INTMOVE(s1, REG_ITMP1);
1237                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1238                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1239                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
1240                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->val.i, REG_ITMP1);
1241                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1242                         emit_store(jd, iptr, iptr->dst, d);
1243                         break;
1244
1245                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1246                                       /* val.l = constant                             */
1247
1248                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1249                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1250                         M_INTMOVE(s1, REG_ITMP1);
1251                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1252                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1253                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
1254                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->val.i), REG_ITMP2);
1255                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
1256                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1257                         emit_store(jd, iptr, iptr->dst, d);
1258                         break;
1259
1260                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1261
1262                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1263                         emit_ishift(cd, SHIFT_SHL, src, iptr);
1264                         break;
1265
1266                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1267                                       /* val.i = constant                             */
1268
1269                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1270                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1271                         M_INTMOVE(s1, d);
1272                         M_ISLL_IMM(iptr->val.i, d);
1273                         emit_store(jd, iptr, iptr->dst, d);
1274                         break;
1275
1276                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1277
1278                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1279                         emit_ishift(cd, SHIFT_SAR, src, iptr);
1280                         break;
1281
1282                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1283                                       /* val.i = constant                             */
1284
1285                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1286                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1287                         M_INTMOVE(s1, d);
1288                         M_ISRA_IMM(iptr->val.i, d);
1289                         emit_store(jd, iptr, iptr->dst, d);
1290                         break;
1291
1292                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1293
1294                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1295                         emit_ishift(cd, SHIFT_SHR, src, iptr);
1296                         break;
1297
1298                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1299                                       /* val.i = constant                             */
1300
1301                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1302                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1303                         M_INTMOVE(s1, d);
1304                         M_ISRL_IMM(iptr->val.i, d);
1305                         emit_store(jd, iptr, iptr->dst, d);
1306                         break;
1307
1308                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1309
1310                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1311                         emit_lshift(cd, SHIFT_SHL, src, iptr);
1312                         break;
1313
1314         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1315                                           /* val.i = constant                             */
1316
1317                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1318                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1319                         M_INTMOVE(s1, d);
1320                         M_LSLL_IMM(iptr->val.i, d);
1321                         emit_store(jd, iptr, iptr->dst, d);
1322                         break;
1323
1324                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1325
1326                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1327                         emit_lshift(cd, SHIFT_SAR, src, iptr);
1328                         break;
1329
1330                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1331                                       /* val.i = constant                             */
1332
1333                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1334                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1335                         M_INTMOVE(s1, d);
1336                         M_LSRA_IMM(iptr->val.i, d);
1337                         emit_store(jd, iptr, iptr->dst, d);
1338                         break;
1339
1340                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1341
1342                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1343                         emit_lshift(cd, SHIFT_SHR, src, iptr);
1344                         break;
1345
1346                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1347                                       /* val.l = constant                             */
1348
1349                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1350                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1351                         M_INTMOVE(s1, d);
1352                         M_LSRL_IMM(iptr->val.i, d);
1353                         emit_store(jd, iptr, iptr->dst, d);
1354                         break;
1355
1356                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1357
1358                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1359                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1360                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1361                         if (s2 == d)
1362                                 M_IAND(s1, d);
1363                         else {
1364                                 M_INTMOVE(s1, d);
1365                                 M_IAND(s2, d);
1366                         }
1367                         emit_store(jd, iptr, iptr->dst, d);
1368                         break;
1369
1370                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1371                                       /* val.i = constant                             */
1372
1373                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1374                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1375                         M_INTMOVE(s1, d);
1376                         M_IAND_IMM(iptr->val.i, d);
1377                         emit_store(jd, iptr, iptr->dst, d);
1378                         break;
1379
1380                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1381
1382                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1383                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1384                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1385                         if (s2 == d)
1386                                 M_LAND(s1, d);
1387                         else {
1388                                 M_INTMOVE(s1, d);
1389                                 M_LAND(s2, d);
1390                         }
1391                         emit_store(jd, iptr, iptr->dst, d);
1392                         break;
1393
1394                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1395                                       /* val.l = constant                             */
1396
1397                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1398                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1399                         M_INTMOVE(s1, d);
1400                         if (IS_IMM32(iptr->val.l))
1401                                 M_LAND_IMM(iptr->val.l, d);
1402                         else {
1403                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
1404                                 M_LAND(REG_ITMP2, d);
1405                         }
1406                         emit_store(jd, iptr, iptr->dst, d);
1407                         break;
1408
1409                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1410
1411                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1412                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1413                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1414                         if (s2 == d)
1415                                 M_IOR(s1, d);
1416                         else {
1417                                 M_INTMOVE(s1, d);
1418                                 M_IOR(s2, d);
1419                         }
1420                         emit_store(jd, iptr, iptr->dst, d);
1421                         break;
1422
1423                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1424                                       /* val.i = constant                             */
1425
1426                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1427                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1428                         M_INTMOVE(s1, d);
1429                         M_IOR_IMM(iptr->val.i, d);
1430                         emit_store(jd, iptr, iptr->dst, d);
1431                         break;
1432
1433                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1434
1435                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1436                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1437                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1438                         if (s2 == d)
1439                                 M_LOR(s1, d);
1440                         else {
1441                                 M_INTMOVE(s1, d);
1442                                 M_LOR(s2, d);
1443                         }
1444                         emit_store(jd, iptr, iptr->dst, d);
1445                         break;
1446
1447                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1448                                       /* val.l = constant                             */
1449
1450                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1451                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1452                         M_INTMOVE(s1, d);
1453                         if (IS_IMM32(iptr->val.l))
1454                                 M_LOR_IMM(iptr->val.l, d);
1455                         else {
1456                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
1457                                 M_LOR(REG_ITMP2, d);
1458                         }
1459                         emit_store(jd, iptr, iptr->dst, d);
1460                         break;
1461
1462                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1463
1464                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1465                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1466                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1467                         if (s2 == d)
1468                                 M_IXOR(s1, d);
1469                         else {
1470                                 M_INTMOVE(s1, d);
1471                                 M_IXOR(s2, d);
1472                         }
1473                         emit_store(jd, iptr, iptr->dst, d);
1474                         break;
1475
1476                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1477                                       /* val.i = constant                             */
1478
1479                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1480                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1481                         M_INTMOVE(s1, d);
1482                         M_IXOR_IMM(iptr->val.i, d);
1483                         emit_store(jd, iptr, iptr->dst, d);
1484                         break;
1485
1486                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1487
1488                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1489                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1490                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
1491                         if (s2 == d)
1492                                 M_LXOR(s1, d);
1493                         else {
1494                                 M_INTMOVE(s1, d);
1495                                 M_LXOR(s2, d);
1496                         }
1497                         emit_store(jd, iptr, iptr->dst, d);
1498                         break;
1499
1500                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1501                                       /* val.l = constant                             */
1502
1503                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1504                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1505                         M_INTMOVE(s1, d);
1506                         if (IS_IMM32(iptr->val.l))
1507                                 M_LXOR_IMM(iptr->val.l, d);
1508                         else {
1509                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
1510                                 M_LXOR(REG_ITMP2, d);
1511                         }
1512                         emit_store(jd, iptr, iptr->dst, d);
1513                         break;
1514
1515
1516                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1517                                       /* op1 = variable, val.i = constant             */
1518
1519                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1520                         if (var->flags & INMEMORY) {
1521                                 s1 = REG_ITMP1;
1522                                 M_ILD(s1, REG_SP, var->regoff * 8);
1523                         } else
1524                                 s1 = var->regoff;
1525
1526                         /* Using inc and dec is not faster than add (tested with
1527                            sieve). */
1528
1529                         M_IADD_IMM(iptr->val.i, s1);
1530
1531                         if (var->flags & INMEMORY)
1532                                 M_IST(s1, REG_SP, var->regoff * 8);
1533                         break;
1534
1535
1536                 /* floating operations ************************************************/
1537
1538                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1539
1540                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1541                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1542                         disp = dseg_adds4(cd, 0x80000000);
1543                         M_FLTMOVE(s1, d);
1544                         emit_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1545                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1546                         emit_store(jd, iptr, iptr->dst, d);
1547                         break;
1548
1549                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1550
1551                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1552                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1553                         disp = dseg_adds8(cd, 0x8000000000000000);
1554                         M_FLTMOVE(s1, d);
1555                         emit_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1556                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1557                         emit_store(jd, iptr, iptr->dst, d);
1558                         break;
1559
1560                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1561
1562                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1563                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1564                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1565                         if (s2 == d)
1566                                 M_FADD(s1, d);
1567                         else {
1568                                 M_FLTMOVE(s1, d);
1569                                 M_FADD(s2, d);
1570                         }
1571                         emit_store(jd, iptr, iptr->dst, d);
1572                         break;
1573
1574                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1575
1576                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1577                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1578                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1579                         if (s2 == d)
1580                                 M_DADD(s1, d);
1581                         else {
1582                                 M_FLTMOVE(s1, d);
1583                                 M_DADD(s2, d);
1584                         }
1585                         emit_store(jd, iptr, iptr->dst, d);
1586                         break;
1587
1588                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1589
1590                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1591                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1592                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1593                         if (s2 == d) {
1594                                 M_FLTMOVE(s2, REG_FTMP2);
1595                                 s2 = REG_FTMP2;
1596                         }
1597                         M_FLTMOVE(s1, d);
1598                         M_FSUB(s2, d);
1599                         emit_store(jd, iptr, iptr->dst, d);
1600                         break;
1601
1602                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1603
1604                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1605                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1606                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1607                         if (s2 == d) {
1608                                 M_FLTMOVE(s2, REG_FTMP2);
1609                                 s2 = REG_FTMP2;
1610                         }
1611                         M_FLTMOVE(s1, d);
1612                         M_DSUB(s2, d);
1613                         emit_store(jd, iptr, iptr->dst, d);
1614                         break;
1615
1616                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1617
1618                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1619                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1620                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1621                         if (s2 == d)
1622                                 M_FMUL(s1, d);
1623                         else {
1624                                 M_FLTMOVE(s1, d);
1625                                 M_FMUL(s2, d);
1626                         }
1627                         emit_store(jd, iptr, iptr->dst, d);
1628                         break;
1629
1630                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1631
1632                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1633                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1634                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1635                         if (s2 == d)
1636                                 M_DMUL(s1, d);
1637                         else {
1638                                 M_FLTMOVE(s1, d);
1639                                 M_DMUL(s2, d);
1640                         }
1641                         emit_store(jd, iptr, iptr->dst, d);
1642                         break;
1643
1644                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1645
1646                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1647                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1648                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1649                         if (s2 == d) {
1650                                 M_FLTMOVE(s2, REG_FTMP2);
1651                                 s2 = REG_FTMP2;
1652                         }
1653                         M_FLTMOVE(s1, d);
1654                         M_FDIV(s2, d);
1655                         emit_store(jd, iptr, iptr->dst, d);
1656                         break;
1657
1658                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1659
1660                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1661                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1662                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1663                         if (s2 == d) {
1664                                 M_FLTMOVE(s2, REG_FTMP2);
1665                                 s2 = REG_FTMP2;
1666                         }
1667                         M_FLTMOVE(s1, d);
1668                         M_DDIV(s2, d);
1669                         emit_store(jd, iptr, iptr->dst, d);
1670                         break;
1671
1672                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1673
1674                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1675                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1676                         M_CVTIF(s1, d);
1677                         emit_store(jd, iptr, iptr->dst, d);
1678                         break;
1679
1680                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1681
1682                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1683                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1684                         M_CVTID(s1, d);
1685                         emit_store(jd, iptr, iptr->dst, d);
1686                         break;
1687
1688                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1689
1690                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1691                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1692                         M_CVTLF(s1, d);
1693                         emit_store(jd, iptr, iptr->dst, d);
1694                         break;
1695
1696                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1697
1698                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1699                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1700                         M_CVTLD(s1, d);
1701                         emit_store(jd, iptr, iptr->dst, d);
1702                         break;
1703
1704                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1705
1706                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1707                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1708                         M_CVTFI(s1, d);
1709                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1710                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1711                                 ((REG_RESULT == d) ? 0 : 3);
1712                         M_BNE(disp);
1713                         M_FLTMOVE(s1, REG_FTMP1);
1714                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1715                         M_CALL(REG_ITMP2);
1716                         M_INTMOVE(REG_RESULT, d);
1717                         emit_store(jd, iptr, iptr->dst, d);
1718                         break;
1719
1720                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1721
1722                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1723                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1724                         M_CVTDI(s1, d);
1725                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1726                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1727                                 ((REG_RESULT == d) ? 0 : 3);
1728                         M_BNE(disp);
1729                         M_FLTMOVE(s1, REG_FTMP1);
1730                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1731                         M_CALL(REG_ITMP2);
1732                         M_INTMOVE(REG_RESULT, d);
1733                         emit_store(jd, iptr, iptr->dst, d);
1734                         break;
1735
1736                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1737
1738                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1739                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1740                         M_CVTFL(s1, d);
1741                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1742                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1743                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1744                                 ((REG_RESULT == d) ? 0 : 3);
1745                         M_BNE(disp);
1746                         M_FLTMOVE(s1, REG_FTMP1);
1747                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1748                         M_CALL(REG_ITMP2);
1749                         M_INTMOVE(REG_RESULT, d);
1750                         emit_store(jd, iptr, iptr->dst, d);
1751                         break;
1752
1753                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1754
1755                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1756                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1757                         M_CVTDL(s1, d);
1758                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1759                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1760                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1761                                 ((REG_RESULT == d) ? 0 : 3);
1762                         M_BNE(disp);
1763                         M_FLTMOVE(s1, REG_FTMP1);
1764                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1765                         M_CALL(REG_ITMP2);
1766                         M_INTMOVE(REG_RESULT, d);
1767                         emit_store(jd, iptr, iptr->dst, d);
1768                         break;
1769
1770                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1771
1772                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1773                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1774                         M_CVTFD(s1, d);
1775                         emit_store(jd, iptr, iptr->dst, d);
1776                         break;
1777
1778                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1779
1780                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1781                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1782                         M_CVTDF(s1, d);
1783                         emit_store(jd, iptr, iptr->dst, d);
1784                         break;
1785
1786                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1787                                           /* == => 0, < => 1, > => -1 */
1788
1789                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1790                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1791                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1792                         M_CLR(d);
1793                         M_MOV_IMM(1, REG_ITMP1);
1794                         M_MOV_IMM(-1, REG_ITMP2);
1795                         emit_ucomiss_reg_reg(cd, s1, s2);
1796                         M_CMOVB(REG_ITMP1, d);
1797                         M_CMOVA(REG_ITMP2, d);
1798                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1799                         emit_store(jd, iptr, iptr->dst, d);
1800                         break;
1801
1802                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1803                                           /* == => 0, < => 1, > => -1 */
1804
1805                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1806                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1807                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1808                         M_CLR(d);
1809                         M_MOV_IMM(1, REG_ITMP1);
1810                         M_MOV_IMM(-1, REG_ITMP2);
1811                         emit_ucomiss_reg_reg(cd, s1, s2);
1812                         M_CMOVB(REG_ITMP1, d);
1813                         M_CMOVA(REG_ITMP2, d);
1814                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1815                         emit_store(jd, iptr, iptr->dst, d);
1816                         break;
1817
1818                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1819                                           /* == => 0, < => 1, > => -1 */
1820
1821                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1822                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1823                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1824                         M_CLR(d);
1825                         M_MOV_IMM(1, REG_ITMP1);
1826                         M_MOV_IMM(-1, REG_ITMP2);
1827                         emit_ucomisd_reg_reg(cd, s1, s2);
1828                         M_CMOVB(REG_ITMP1, d);
1829                         M_CMOVA(REG_ITMP2, d);
1830                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1831                         emit_store(jd, iptr, iptr->dst, d);
1832                         break;
1833
1834                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1835                                           /* == => 0, < => 1, > => -1 */
1836
1837                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1838                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1839                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1840                         M_CLR(d);
1841                         M_MOV_IMM(1, REG_ITMP1);
1842                         M_MOV_IMM(-1, REG_ITMP2);
1843                         emit_ucomisd_reg_reg(cd, s1, s2);
1844                         M_CMOVB(REG_ITMP1, d);
1845                         M_CMOVA(REG_ITMP2, d);
1846                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1847                         emit_store(jd, iptr, iptr->dst, d);
1848                         break;
1849
1850
1851                 /* memory operations **************************************************/
1852
1853                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1854
1855                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1856                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1857                         gen_nullptr_check(s1);
1858                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1859                         emit_store(jd, iptr, iptr->dst, d);
1860                         break;
1861
1862                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1863
1864                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1865                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1866                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1867                         if (iptr->op1 == 0) {
1868                                 gen_nullptr_check(s1);
1869                                 gen_bound_check;
1870                         }
1871                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1872                         emit_store(jd, iptr, iptr->dst, d);
1873                         break;
1874
1875                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1876
1877                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1878                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1879                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1880                         if (iptr->op1 == 0) {
1881                                 gen_nullptr_check(s1);
1882                                 gen_bound_check;
1883                         }
1884                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1885                         emit_store(jd, iptr, iptr->dst, d);
1886                         break;
1887
1888                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1889
1890                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1891                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1892                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1893                         if (iptr->op1 == 0) {
1894                                 gen_nullptr_check(s1);
1895                                 gen_bound_check;
1896                         }
1897                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1898                         emit_store(jd, iptr, iptr->dst, d);
1899                         break;
1900
1901                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1902
1903                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1904                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1905                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1906                         if (iptr->op1 == 0) {
1907                                 gen_nullptr_check(s1);
1908                                 gen_bound_check;
1909                         }
1910                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1911                         emit_store(jd, iptr, iptr->dst, d);
1912                         break;
1913
1914                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1915
1916                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1917                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1918                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1919                         if (iptr->op1 == 0) {
1920                                 gen_nullptr_check(s1);
1921                                 gen_bound_check;
1922                         }
1923                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1924                         emit_store(jd, iptr, iptr->dst, d);
1925                         break;
1926
1927                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1928
1929                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1930                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1931                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1932                         if (iptr->op1 == 0) {
1933                                 gen_nullptr_check(s1);
1934                                 gen_bound_check;
1935                         }
1936                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1937                         emit_store(jd, iptr, iptr->dst, d);
1938                         break;
1939
1940                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1941
1942                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1943                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1944                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1945                         if (iptr->op1 == 0) {
1946                                 gen_nullptr_check(s1);
1947                                 gen_bound_check;
1948                         }
1949                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1950                         emit_store(jd, iptr, iptr->dst, d);
1951                         break;
1952
1953                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1954
1955                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
1956                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
1957                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1958                         if (iptr->op1 == 0) {
1959                                 gen_nullptr_check(s1);
1960                                 gen_bound_check;
1961                         }
1962                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1963                         emit_store(jd, iptr, iptr->dst, d);
1964                         break;
1965
1966
1967                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1968
1969                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
1970                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
1971                         if (iptr->op1 == 0) {
1972                                 gen_nullptr_check(s1);
1973                                 gen_bound_check;
1974                         }
1975                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
1976                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1977                         break;
1978
1979                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1980
1981                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
1982                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
1983                         if (iptr->op1 == 0) {
1984                                 gen_nullptr_check(s1);
1985                                 gen_bound_check;
1986                         }
1987                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
1988                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1989                         break;
1990
1991                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1992
1993                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
1994                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
1995                         if (iptr->op1 == 0) {
1996                                 gen_nullptr_check(s1);
1997                                 gen_bound_check;
1998                         }
1999                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2000                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2001                         break;
2002
2003                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2004
2005                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2006                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2007                         if (iptr->op1 == 0) {
2008                                 gen_nullptr_check(s1);
2009                                 gen_bound_check;
2010                         }
2011                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2012                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2013                         break;
2014
2015                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2016
2017                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2018                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2019                         if (iptr->op1 == 0) {
2020                                 gen_nullptr_check(s1);
2021                                 gen_bound_check;
2022                         }
2023                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2024                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2025                         break;
2026
2027                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2028
2029                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2030                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2031                         if (iptr->op1 == 0) {
2032                                 gen_nullptr_check(s1);
2033                                 gen_bound_check;
2034                         }
2035                         s3 = emit_load_s3(jd, iptr, src, REG_FTMP3);
2036                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2037                         break;
2038
2039                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2040
2041                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2042                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2043                         if (iptr->op1 == 0) {
2044                                 gen_nullptr_check(s1);
2045                                 gen_bound_check;
2046                         }
2047                         s3 = emit_load_s3(jd, iptr, src, REG_FTMP3);
2048                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2049                         break;
2050
2051                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2052
2053                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2054                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2055                         if (iptr->op1 == 0) {
2056                                 gen_nullptr_check(s1);
2057                                 gen_bound_check;
2058                         }
2059                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2060
2061                         M_MOV(s1, rd->argintregs[0]);
2062                         M_MOV(s3, rd->argintregs[1]);
2063                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
2064                         M_CALL(REG_ITMP1);
2065                         M_TEST(REG_RESULT);
2066                         M_BEQ(0);
2067                         codegen_add_arraystoreexception_ref(cd);
2068
2069                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2070                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2071                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2072                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2073                         break;
2074
2075
2076                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2077
2078                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2079                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2080                         if (iptr->op1 == 0) {
2081                                 gen_nullptr_check(s1);
2082                                 gen_bound_check;
2083                         }
2084                         emit_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2085                         break;
2086
2087                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2088
2089                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2090                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2091                         if (iptr->op1 == 0) {
2092                                 gen_nullptr_check(s1);
2093                                 gen_bound_check;
2094                         }
2095                         emit_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2096                         break;
2097
2098                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2099
2100                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2101                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2102                         if (iptr->op1 == 0) {
2103                                 gen_nullptr_check(s1);
2104                                 gen_bound_check;
2105                         }
2106                         emit_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2107                         break;
2108
2109                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2110
2111                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2112                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2113                         if (iptr->op1 == 0) {
2114                                 gen_nullptr_check(s1);
2115                                 gen_bound_check;
2116                         }
2117                         emit_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2118                         break;
2119
2120                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2121
2122                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2123                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2124                         if (iptr->op1 == 0) {
2125                                 gen_nullptr_check(s1);
2126                                 gen_bound_check;
2127                         }
2128
2129                         if (IS_IMM32(iptr->val.l)) {
2130                                 emit_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2131                         } else {
2132                                 emit_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2133                                 emit_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2134                         }
2135                         break;
2136
2137                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2138
2139                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2140                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2141                         if (iptr->op1 == 0) {
2142                                 gen_nullptr_check(s1);
2143                                 gen_bound_check;
2144                         }
2145                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2146                         break;
2147
2148
2149                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2150                                       /* op1 = type, val.a = field address            */
2151
2152                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2153                                 disp = dseg_addaddress(cd, NULL);
2154
2155 /*                              PROFILE_CYCLE_STOP; */
2156
2157                                 codegen_addpatchref(cd, PATCHER_get_putstatic,
2158                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), disp);
2159
2160                                 if (opt_showdisassemble) {
2161                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2162                                 }
2163
2164 /*                              PROFILE_CYCLE_START; */
2165
2166                         } else {
2167                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr);
2168
2169                                 disp = dseg_addaddress(cd, &(fi->value));
2170
2171                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2172                                         PROFILE_CYCLE_STOP;
2173
2174                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2175
2176                                         if (opt_showdisassemble) {
2177                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2178                                         }
2179
2180                                         PROFILE_CYCLE_START;
2181                                 }
2182                         }
2183
2184                         /* This approach is much faster than moving the field
2185                            address inline into a register. */
2186
2187                         M_ALD(REG_ITMP1, RIP, -(((ptrint) cd->mcodeptr + 7) -
2188                                                                         (ptrint) cd->mcodebase) + disp);
2189
2190                         switch (iptr->op1) {
2191                         case TYPE_INT:
2192                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
2193                                 M_ILD(d, REG_ITMP1, 0);
2194                                 break;
2195                         case TYPE_LNG:
2196                         case TYPE_ADR:
2197                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
2198                                 M_LLD(d, REG_ITMP1, 0);
2199                                 break;
2200                         case TYPE_FLT:
2201                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2202                                 M_FLD(d, REG_ITMP1, 0);
2203                                 break;
2204                         case TYPE_DBL:
2205                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2206                                 M_DLD(d, REG_ITMP1, 0);
2207                                 break;
2208                         }
2209                         emit_store(jd, iptr, iptr->dst, d);
2210                         break;
2211
2212                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2213                                       /* op1 = type, val.a = field address            */
2214
2215                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2216                                 disp = dseg_addaddress(cd, NULL);
2217
2218 /*                              PROFILE_CYCLE_STOP; */
2219
2220                                 codegen_addpatchref(cd, PATCHER_get_putstatic,
2221                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), disp);
2222
2223                                 if (opt_showdisassemble) {
2224                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2225                                 }
2226
2227 /*                              PROFILE_CYCLE_START; */
2228
2229                         } else {
2230                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr);
2231
2232                                 disp = dseg_addaddress(cd, &(fi->value));
2233
2234                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2235                                         PROFILE_CYCLE_STOP;
2236
2237                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2238
2239                                         if (opt_showdisassemble) {
2240                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2241                                         }
2242
2243                                         PROFILE_CYCLE_START;
2244                                 }
2245                         }
2246
2247                         /* This approach is much faster than moving the field
2248                            address inline into a register. */
2249
2250                         M_ALD(REG_ITMP1, RIP, -(((ptrint) cd->mcodeptr + 7) -
2251                                                                         (ptrint) cd->mcodebase) + disp);
2252
2253                         switch (iptr->op1) {
2254                         case TYPE_INT:
2255                                 s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2256                                 M_IST(s2, REG_ITMP1, 0);
2257                                 break;
2258                         case TYPE_LNG:
2259                         case TYPE_ADR:
2260                                 s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2261                                 M_LST(s2, REG_ITMP1, 0);
2262                                 break;
2263                         case TYPE_FLT:
2264                                 s2 = emit_load_s2(jd, iptr, src, REG_FTMP1);
2265                                 M_FST(s2, REG_ITMP1, 0);
2266                                 break;
2267                         case TYPE_DBL:
2268                                 s2 = emit_load_s2(jd, iptr, src, REG_FTMP1);
2269                                 M_DST(s2, REG_ITMP1, 0);
2270                                 break;
2271                         }
2272                         break;
2273
2274                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2275                                           /* val = value (in current instruction)     */
2276                                           /* op1 = type, val.a = field address (in    */
2277                                           /* following NOP)                           */
2278
2279                         if (INSTRUCTION_IS_UNRESOLVED(iptr + 1)) {
2280                                 disp = dseg_addaddress(cd, NULL);
2281
2282 /*                              PROFILE_CYCLE_STOP; */
2283
2284                                 codegen_addpatchref(cd, PATCHER_get_putstatic,
2285                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr + 1), disp);
2286
2287                                 if (opt_showdisassemble) {
2288                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2289                                 }
2290
2291 /*                              PROFILE_CYCLE_START; */
2292
2293                         } else {
2294                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr + 1);
2295
2296                                 disp = dseg_addaddress(cd, &(fi->value));
2297
2298                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2299                                         PROFILE_CYCLE_STOP;
2300
2301                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2302
2303                                         if (opt_showdisassemble) {
2304                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2305                                         }
2306
2307                                         PROFILE_CYCLE_START;
2308                                 }
2309                         }
2310
2311                         /* This approach is much faster than moving the field
2312                            address inline into a register. */
2313
2314                         M_ALD(REG_ITMP1, RIP, -(((ptrint) cd->mcodeptr + 7) -
2315                                                                         (ptrint) cd->mcodebase) + disp);
2316
2317                         switch (iptr->op1) {
2318                         case TYPE_INT:
2319                         case TYPE_FLT:
2320                                 M_IST_IMM(iptr->val.i, REG_ITMP1, 0);
2321                                 break;
2322                         case TYPE_LNG:
2323                         case TYPE_ADR:
2324                         case TYPE_DBL:
2325                                 if (IS_IMM32(iptr->val.l))
2326                                         M_LST_IMM32(iptr->val.l, REG_ITMP1, 0);
2327                                 else {
2328                                         M_IST_IMM(iptr->val.l, REG_ITMP1, 0);
2329                                         M_IST_IMM(iptr->val.l >> 32, REG_ITMP1, 4);
2330                                 }
2331                                 break;
2332                         }
2333                         break;
2334
2335                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2336                                       /* op1 = type, val.i = field offset             */
2337
2338                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2339                         gen_nullptr_check(s1);
2340
2341                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2342 /*                              PROFILE_CYCLE_STOP; */
2343
2344                                 codegen_addpatchref(cd, PATCHER_get_putfield,
2345                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), 0);
2346
2347                                 if (opt_showdisassemble) {
2348                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2349                                 }
2350
2351 /*                              PROFILE_CYCLE_START; */
2352
2353                                 disp = 0;
2354
2355                         } else
2356                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr)->offset;
2357
2358                         switch (iptr->op1) {
2359                         case TYPE_INT:
2360                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2361                                 M_ILD32(d, s1, disp);
2362                                 break;
2363                         case TYPE_LNG:
2364                         case TYPE_ADR:
2365                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2366                                 M_LLD32(d, s1, disp);
2367                                 break;
2368                         case TYPE_FLT:
2369                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2370                                 M_FLD32(d, s1, disp);
2371                                 break;
2372                         case TYPE_DBL:
2373                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2374                                 M_DLD32(d, s1, disp);
2375                                 break;
2376                         }
2377                         emit_store(jd, iptr, iptr->dst, d);
2378                         break;
2379
2380                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2381                                       /* op1 = type, val.i = field offset             */
2382
2383                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2384                         gen_nullptr_check(s1);
2385
2386                         if (IS_INT_LNG_TYPE(iptr->op1))
2387                                 s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2388                         else
2389                                 s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
2390
2391                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2392 /*                              PROFILE_CYCLE_STOP; */
2393
2394                                 codegen_addpatchref(cd, PATCHER_get_putfield,
2395                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), 0);
2396
2397                                 if (opt_showdisassemble) {
2398                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2399                                 }
2400
2401 /*                              PROFILE_CYCLE_START; */
2402
2403                                 disp = 0;
2404
2405                         } else
2406                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr)->offset;
2407
2408                         switch (iptr->op1) {
2409                         case TYPE_INT:
2410                                 M_IST32(s2, s1, disp);
2411                                 break;
2412                         case TYPE_LNG:
2413                         case TYPE_ADR:
2414                                 M_LST32(s2, s1, disp);
2415                                 break;
2416                         case TYPE_FLT:
2417                                 M_FST32(s2, s1, disp);
2418                                 break;
2419                         case TYPE_DBL:
2420                                 M_DST32(s2, s1, disp);
2421                                 break;
2422                         }
2423                         break;
2424
2425                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2426                                           /* val = value (in current instruction)     */
2427                                           /* op1 = type, val.a = field address (in    */
2428                                           /* following NOP)                           */
2429
2430                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2431                         gen_nullptr_check(s1);
2432
2433                         if (INSTRUCTION_IS_UNRESOLVED(iptr + 1)) {
2434 /*                              PROFILE_CYCLE_STOP; */
2435
2436                                 codegen_addpatchref(cd, PATCHER_putfieldconst,
2437                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr + 1), 0);
2438
2439                                 if (opt_showdisassemble) {
2440                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2441                                 }
2442
2443 /*                              PROFILE_CYCLE_START; */
2444
2445                                 disp = 0;
2446
2447                         } else
2448                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr + 1)->offset;
2449
2450                         switch (iptr->op1) {
2451                         case TYPE_INT:
2452                         case TYPE_FLT:
2453                                 M_IST32_IMM(iptr->val.i, s1, disp);
2454                                 break;
2455                         case TYPE_LNG:
2456                         case TYPE_ADR:
2457                         case TYPE_DBL:
2458                                 M_IST32_IMM(iptr->val.l, s1, disp);
2459                                 M_IST32_IMM(iptr->val.l >> 32, s1, disp + 4);
2460                                 break;
2461                         }
2462                         break;
2463
2464
2465                 /* branch operations **************************************************/
2466
2467                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2468
2469                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2470                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2471
2472                         PROFILE_CYCLE_STOP;
2473
2474 #ifdef ENABLE_VERIFIER
2475                         if (iptr->val.a) {
2476                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2477                                                                         (unresolved_class *) iptr->val.a, 0);
2478
2479                                 if (opt_showdisassemble) {
2480                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2481                                 }
2482                         }
2483 #endif /* ENABLE_VERIFIER */
2484
2485                         M_CALL_IMM(0);                            /* passing exception pc */
2486                         M_POP(REG_ITMP2_XPC);
2487
2488                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2489                         M_JMP(REG_ITMP3);
2490                         break;
2491
2492                 case ICMD_GOTO:         /* ... ==> ...                                */
2493                                         /* op1 = target JavaVM pc                     */
2494
2495                         M_JMP_IMM(0);
2496                         codegen_addreference(cd, (basicblock *) iptr->target);
2497                         break;
2498
2499                 case ICMD_JSR:          /* ... ==> ...                                */
2500                                         /* op1 = target JavaVM pc                     */
2501
2502                         M_CALL_IMM(0);
2503                         codegen_addreference(cd, (basicblock *) iptr->target);
2504                         break;
2505
2506                 case ICMD_RET:          /* ... ==> ...                                */
2507                                         /* op1 = local variable                       */
2508
2509                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2510                         if (var->flags & INMEMORY) {
2511                                 M_ALD(REG_ITMP1, REG_SP, var->regoff * 8);
2512                                 M_JMP(REG_ITMP1);
2513                         } else
2514                                 M_JMP(var->regoff);
2515                         break;
2516
2517                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2518                                         /* op1 = target JavaVM pc                     */
2519
2520                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2521                         M_TEST(s1);
2522                         M_BEQ(0);
2523                         codegen_addreference(cd, (basicblock *) iptr->target);
2524                         break;
2525
2526                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2527                                         /* op1 = target JavaVM pc                     */
2528
2529                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2530                         M_TEST(s1);
2531                         M_BNE(0);
2532                         codegen_addreference(cd, (basicblock *) iptr->target);
2533                         break;
2534
2535                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2536                                         /* op1 = target JavaVM pc, val.i = constant   */
2537
2538                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2539                         M_ICMP_IMM(iptr->val.i, s1);
2540                         M_BEQ(0);
2541                         codegen_addreference(cd, (basicblock *) iptr->target);
2542                         break;
2543
2544                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2545                                         /* op1 = target JavaVM pc, val.i = constant   */
2546
2547                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2548                         M_ICMP_IMM(iptr->val.i, s1);
2549                         M_BLT(0);
2550                         codegen_addreference(cd, (basicblock *) iptr->target);
2551                         break;
2552
2553                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2554                                         /* op1 = target JavaVM pc, val.i = constant   */
2555
2556                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2557                         M_ICMP_IMM(iptr->val.i, s1);
2558                         M_BLE(0);
2559                         codegen_addreference(cd, (basicblock *) iptr->target);
2560                         break;
2561
2562                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2563                                         /* op1 = target JavaVM pc, val.i = constant   */
2564
2565                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2566                         M_ICMP_IMM(iptr->val.i, s1);
2567                         M_BNE(0);
2568                         codegen_addreference(cd, (basicblock *) iptr->target);
2569                         break;
2570
2571                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2572                                         /* op1 = target JavaVM pc, val.i = constant   */
2573
2574                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2575                         M_ICMP_IMM(iptr->val.i, s1);
2576                         M_BGT(0);
2577                         codegen_addreference(cd, (basicblock *) iptr->target);
2578                         break;
2579
2580                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2581                                         /* op1 = target JavaVM pc, val.i = constant   */
2582
2583                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2584                         M_ICMP_IMM(iptr->val.i, s1);
2585                         M_BGE(0);
2586                         codegen_addreference(cd, (basicblock *) iptr->target);
2587                         break;
2588
2589                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2590                                         /* op1 = target JavaVM pc, val.l = constant   */
2591
2592                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2593                         if (IS_IMM32(iptr->val.l))
2594                                 M_LCMP_IMM(iptr->val.l, s1);
2595                         else {
2596                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
2597                                 M_LCMP(REG_ITMP2, s1);
2598                         }
2599                         M_BEQ(0);
2600                         codegen_addreference(cd, (basicblock *) iptr->target);
2601                         break;
2602
2603                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2604                                         /* op1 = target JavaVM pc, val.l = constant   */
2605
2606                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2607                         if (IS_IMM32(iptr->val.l))
2608                                 M_LCMP_IMM(iptr->val.l, s1);
2609                         else {
2610                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
2611                                 M_LCMP(REG_ITMP2, s1);
2612                         }
2613                         M_BLT(0);
2614                         codegen_addreference(cd, (basicblock *) iptr->target);
2615                         break;
2616
2617                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2618                                         /* op1 = target JavaVM pc, val.l = constant   */
2619
2620                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2621                         if (IS_IMM32(iptr->val.l))
2622                                 M_LCMP_IMM(iptr->val.l, s1);
2623                         else {
2624                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
2625                                 M_LCMP(REG_ITMP2, s1);
2626                         }
2627                         M_BLE(0);
2628                         codegen_addreference(cd, (basicblock *) iptr->target);
2629                         break;
2630
2631                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2632                                         /* op1 = target JavaVM pc, val.l = constant   */
2633
2634                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2635                         if (IS_IMM32(iptr->val.l))
2636                                 M_LCMP_IMM(iptr->val.l, s1);
2637                         else {
2638                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
2639                                 M_LCMP(REG_ITMP2, s1);
2640                         }
2641                         M_BNE(0);
2642                         codegen_addreference(cd, (basicblock *) iptr->target);
2643                         break;
2644
2645                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2646                                         /* op1 = target JavaVM pc, val.l = constant   */
2647
2648                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2649                         if (IS_IMM32(iptr->val.l))
2650                                 M_LCMP_IMM(iptr->val.l, s1);
2651                         else {
2652                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
2653                                 M_LCMP(REG_ITMP2, s1);
2654                         }
2655                         M_BGT(0);
2656                         codegen_addreference(cd, (basicblock *) iptr->target);
2657                         break;
2658
2659                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2660                                         /* op1 = target JavaVM pc, val.l = constant   */
2661
2662                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2663                         if (IS_IMM32(iptr->val.l))
2664                                 M_LCMP_IMM(iptr->val.l, s1);
2665                         else {
2666                                 M_MOV_IMM(iptr->val.l, REG_ITMP2);
2667                                 M_LCMP(REG_ITMP2, s1);
2668                         }
2669                         M_BGE(0);
2670                         codegen_addreference(cd, (basicblock *) iptr->target);
2671                         break;
2672
2673                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2674                                         /* op1 = target JavaVM pc                     */
2675
2676                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2677                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2678                         M_ICMP(s2, s1);
2679                         M_BEQ(0);
2680                         codegen_addreference(cd, (basicblock *) iptr->target);
2681                         break;
2682
2683                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2684                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2685
2686                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2687                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2688                         M_LCMP(s2, s1);
2689                         M_BEQ(0);
2690                         codegen_addreference(cd, (basicblock *) iptr->target);
2691                         break;
2692
2693                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2694                                         /* op1 = target JavaVM pc                     */
2695
2696                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2697                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2698                         M_ICMP(s2, s1);
2699                         M_BNE(0);
2700                         codegen_addreference(cd, (basicblock *) iptr->target);
2701                         break;
2702
2703                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2704                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2705
2706                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2707                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2708                         M_LCMP(s2, s1);
2709                         M_BNE(0);
2710                         codegen_addreference(cd, (basicblock *) iptr->target);
2711                         break;
2712
2713                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2714                                         /* op1 = target JavaVM pc                     */
2715
2716                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2717                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2718                         M_ICMP(s2, s1);
2719                         M_BLT(0);
2720                         codegen_addreference(cd, (basicblock *) iptr->target);
2721                         break;
2722
2723                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2724                                     /* op1 = target JavaVM pc                     */
2725
2726                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2727                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2728                         M_LCMP(s2, s1);
2729                         M_BLT(0);
2730                         codegen_addreference(cd, (basicblock *) iptr->target);
2731                         break;
2732
2733                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2734                                         /* op1 = target JavaVM pc                     */
2735
2736                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2737                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2738                         M_ICMP(s2, s1);
2739                         M_BGT(0);
2740                         codegen_addreference(cd, (basicblock *) iptr->target);
2741                         break;
2742
2743                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2744                                 /* op1 = target JavaVM pc                     */
2745
2746                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2747                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2748                         M_LCMP(s2, s1);
2749                         M_BGT(0);
2750                         codegen_addreference(cd, (basicblock *) iptr->target);
2751                         break;
2752
2753                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2754                                         /* op1 = target JavaVM pc                     */
2755
2756                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2757                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2758                         M_ICMP(s2, s1);
2759                         M_BLE(0);
2760                         codegen_addreference(cd, (basicblock *) iptr->target);
2761                         break;
2762
2763                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2764                                         /* op1 = target JavaVM pc                     */
2765
2766                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2767                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2768                         M_LCMP(s2, s1);
2769                         M_BLE(0);
2770                         codegen_addreference(cd, (basicblock *) iptr->target);
2771                         break;
2772
2773                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2774                                         /* op1 = target JavaVM pc                     */
2775
2776                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2777                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2778                         M_ICMP(s2, s1);
2779                         M_BGE(0);
2780                         codegen_addreference(cd, (basicblock *) iptr->target);
2781                         break;
2782
2783                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2784                                     /* op1 = target JavaVM pc                     */
2785
2786                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2787                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2788                         M_LCMP(s2, s1);
2789                         M_BGE(0);
2790                         codegen_addreference(cd, (basicblock *) iptr->target);
2791                         break;
2792
2793
2794                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2795                 case ICMD_LRETURN:
2796
2797                         s1 = emit_load_s1(jd, iptr, src, REG_RESULT);
2798                         M_INTMOVE(s1, REG_RESULT);
2799                         goto nowperformreturn;
2800
2801                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2802
2803                         s1 = emit_load_s1(jd, iptr, src, REG_RESULT);
2804                         M_INTMOVE(s1, REG_RESULT);
2805
2806 #ifdef ENABLE_VERIFIER
2807                         if (iptr->val.a) {
2808                                 PROFILE_CYCLE_STOP;
2809
2810                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2811                                                                         (unresolved_class *) iptr->val.a, 0);
2812
2813                                 if (opt_showdisassemble) {
2814                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2815                                 }
2816
2817                                 PROFILE_CYCLE_START;
2818                         }
2819 #endif /* ENABLE_VERIFIER */
2820                         goto nowperformreturn;
2821
2822                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2823                 case ICMD_DRETURN:
2824
2825                         s1 = emit_load_s1(jd, iptr, src, REG_FRESULT);
2826                         M_FLTMOVE(s1, REG_FRESULT);
2827                         goto nowperformreturn;
2828
2829                 case ICMD_RETURN:      /* ...  ==> ...                                */
2830
2831 nowperformreturn:
2832                         {
2833                         s4 i, p;
2834
2835                         p = stackframesize;
2836
2837 #if !defined(NDEBUG)
2838                         /* generate call trace */
2839
2840                         if (opt_verbosecall) {
2841                                 emit_alu_imm_reg(cd, ALU_SUB, 2 * 8, REG_SP);
2842
2843                                 emit_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2844                                 emit_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2845
2846                                 emit_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
2847                                 emit_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2848                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2849                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2850
2851                                 M_MOV_IMM(builtin_displaymethodstop, REG_ITMP1);
2852                                 M_CALL(REG_ITMP1);
2853
2854                                 emit_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2855                                 emit_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2856
2857                                 emit_alu_imm_reg(cd, ALU_ADD, 2 * 8, REG_SP);
2858                         }
2859 #endif /* !defined(NDEBUG) */
2860
2861 #if defined(ENABLE_THREADS)
2862                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2863                                 M_ALD(rd->argintregs[0], REG_SP, rd->memuse * 8);
2864
2865                                 /* we need to save the proper return value */
2866                                 switch (iptr->opc) {
2867                                 case ICMD_IRETURN:
2868                                 case ICMD_ARETURN:
2869                                 case ICMD_LRETURN:
2870                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2871                                         break;
2872                                 case ICMD_FRETURN:
2873                                 case ICMD_DRETURN:
2874                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2875                                         break;
2876                                 }
2877
2878                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2879                                 M_CALL(REG_ITMP1);
2880
2881                                 /* and now restore the proper return value */
2882                                 switch (iptr->opc) {
2883                                 case ICMD_IRETURN:
2884                                 case ICMD_ARETURN:
2885                                 case ICMD_LRETURN:
2886                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2887                                         break;
2888                                 case ICMD_FRETURN:
2889                                 case ICMD_DRETURN:
2890                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2891                                         break;
2892                                 }
2893                         }
2894 #endif
2895
2896                         /* restore saved registers */
2897
2898                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2899                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2900                         }
2901                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2902                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2903                         }
2904
2905                         /* deallocate stack */
2906
2907                         if (stackframesize)
2908                                 M_AADD_IMM(stackframesize * 8, REG_SP);
2909
2910                         /* generate method profiling code */
2911
2912                         PROFILE_CYCLE_STOP;
2913
2914                         M_RET;
2915                         }
2916                         break;
2917
2918
2919                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2920                         {
2921                                 s4 i, l, *s4ptr;
2922                                 void **tptr;
2923
2924                                 tptr = (void **) iptr->target;
2925
2926                                 s4ptr = iptr->val.a;
2927                                 l = s4ptr[1];                          /* low     */
2928                                 i = s4ptr[2];                          /* high    */
2929
2930                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2931                                 M_INTMOVE(s1, REG_ITMP1);
2932
2933                                 if (l != 0)
2934                                         M_ISUB_IMM(l, REG_ITMP1);
2935
2936                                 i = i - l + 1;
2937
2938                 /* range check */
2939                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2940                                 M_BA(0);
2941
2942                                 codegen_addreference(cd, (basicblock *) tptr[0]);
2943
2944                                 /* build jump table top down and use address of lowest entry */
2945
2946                 /* s4ptr += 3 + i; */
2947                                 tptr += i;
2948
2949                                 while (--i >= 0) {
2950                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
2951                                         --tptr;
2952                                 }
2953
2954                                 /* length of dataseg after last dseg_addtarget is used
2955                                    by load */
2956
2957                                 M_MOV_IMM(0, REG_ITMP2);
2958                                 dseg_adddata(cd);
2959                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2960                                 M_JMP(REG_ITMP1);
2961                         }
2962                         break;
2963
2964
2965                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2966                         {
2967                                 s4 i, l, val, *s4ptr;
2968                                 void **tptr;
2969
2970                                 tptr = (void **) iptr->target;
2971
2972                                 s4ptr = iptr->val.a;
2973                                 l = s4ptr[0];                          /* default  */
2974                                 i = s4ptr[1];                          /* count    */
2975
2976                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2977                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2978
2979                                 while (--i >= 0) {
2980                                         s4ptr += 2;
2981                                         ++tptr;
2982
2983                                         val = s4ptr[0];
2984                                         M_ICMP_IMM(val, s1);
2985                                         M_BEQ(0);
2986                                         codegen_addreference(cd, (basicblock *) tptr[0]);
2987                                 }
2988
2989                                 M_JMP_IMM(0);
2990
2991                                 tptr = (void **) iptr->target;
2992                                 codegen_addreference(cd, (basicblock *) tptr[0]);
2993                         }
2994                         break;
2995
2996
2997                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2998                                         /* op1 = arg count val.a = builtintable entry */
2999
3000                         bte = iptr->val.a;
3001                         md = bte->md;
3002                         goto gen_method;
3003
3004                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
3005                                         /* op1 = arg count, val.a = method pointer    */
3006
3007                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
3008                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
3009                 case ICMD_INVOKEINTERFACE:
3010
3011                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3012                                 md = INSTRUCTION_UNRESOLVED_METHOD(iptr)->methodref->parseddesc.md;
3013                                 lm = NULL;
3014                         }
3015                         else {
3016                                 lm = INSTRUCTION_RESOLVED_METHODINFO(iptr);
3017                                 md = lm->parseddesc;
3018                         }
3019
3020 gen_method:
3021                         s3 = md->paramcount;
3022
3023                         MCODECHECK((20 * s3) + 128);
3024
3025                         /* copy arguments to registers or stack location */
3026
3027                         for (s3 = s3 - 1; s3 >= 0; s3--, src = src->prev) {
3028                                 if (src->varkind == ARGVAR)
3029                                         continue;
3030                                 if (IS_INT_LNG_TYPE(src->type)) {
3031                                         if (!md->params[s3].inmemory) {
3032                                                 s1 = rd->argintregs[md->params[s3].regoff];
3033                                                 d = emit_load_s1(jd, iptr, src, s1);
3034                                                 M_INTMOVE(d, s1);
3035                                         } else {
3036                                                 d = emit_load_s1(jd, iptr, src, REG_ITMP1);
3037                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
3038                                         }
3039
3040                                 } else {
3041                                         if (!md->params[s3].inmemory) {
3042                                                 s1 = rd->argfltregs[md->params[s3].regoff];
3043                                                 d = emit_load_s1(jd, iptr, src, s1);
3044                                                 M_FLTMOVE(d, s1);
3045                                         } else {
3046                                                 d = emit_load_s1(jd, iptr, src, REG_FTMP1);
3047
3048                                                 if (IS_2_WORD_TYPE(src->type))
3049                                                         M_DST(d, REG_SP, md->params[s3].regoff * 8);
3050                                                 else
3051                                                         M_FST(d, REG_SP, md->params[s3].regoff * 8);
3052                                         }
3053                                 }
3054                         }
3055
3056                         /* generate method profiling code */
3057
3058                         PROFILE_CYCLE_STOP;
3059
3060                         switch (iptr->opc) {
3061                         case ICMD_BUILTIN:
3062                                 a = (ptrint) bte->fp;
3063                                 d = md->returntype.type;
3064
3065                                 M_MOV_IMM(a, REG_ITMP1);
3066                                 M_CALL(REG_ITMP1);
3067
3068                                 /* if op1 == true, we need to check for an exception */
3069
3070                                 if (iptr->op1 == true) {
3071                                         M_TEST(REG_RESULT);
3072                                         M_BEQ(0);
3073                                         codegen_add_fillinstacktrace_ref(cd);
3074                                 }
3075                                 break;
3076
3077                         case ICMD_INVOKESPECIAL:
3078                                 M_TEST(rd->argintregs[0]);
3079                                 M_BEQ(0);
3080                                 codegen_add_nullpointerexception_ref(cd);
3081
3082                                 /* first argument contains pointer */
3083 /*                              gen_nullptr_check(rd->argintregs[0]); */
3084
3085                                 /* access memory for hardware nullptr */
3086 /*                              emit_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); */
3087
3088                                 /* fall through */
3089
3090                         case ICMD_INVOKESTATIC:
3091                                 if (lm == NULL) {
3092                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3093
3094                                         codegen_addpatchref(cd, PATCHER_invokestatic_special,
3095                                                                                 um, 0);
3096
3097                                         if (opt_showdisassemble) {
3098                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3099                                         }
3100
3101                                         a = 0;
3102                                         d = um->methodref->parseddesc.md->returntype.type;
3103
3104                                 } else {
3105                                         a = (ptrint) lm->stubroutine;
3106                                         d = lm->parseddesc->returntype.type;
3107                                 }
3108
3109                                 M_MOV_IMM(a, REG_ITMP2);
3110                                 M_CALL(REG_ITMP2);
3111                                 break;
3112
3113                         case ICMD_INVOKEVIRTUAL:
3114                                 gen_nullptr_check(rd->argintregs[0]);
3115
3116                                 if (lm == NULL) {
3117                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3118
3119                                         codegen_addpatchref(cd, PATCHER_invokevirtual, um, 0);
3120
3121                                         if (opt_showdisassemble) {
3122                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3123                                         }
3124
3125                                         s1 = 0;
3126                                         d = um->methodref->parseddesc.md->returntype.type;
3127
3128                                 } else {
3129                                         s1 = OFFSET(vftbl_t, table[0]) +
3130                                                 sizeof(methodptr) * lm->vftblindex;
3131                                         d = lm->parseddesc->returntype.type;
3132                                 }
3133
3134                                 M_ALD(REG_METHODPTR, rd->argintregs[0],
3135                                           OFFSET(java_objectheader, vftbl));
3136                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
3137                                 M_CALL(REG_ITMP3);
3138                                 break;
3139
3140                         case ICMD_INVOKEINTERFACE:
3141                                 gen_nullptr_check(rd->argintregs[0]);
3142
3143                                 if (lm == NULL) {
3144                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3145
3146                                         codegen_addpatchref(cd, PATCHER_invokeinterface, um, 0);
3147
3148                                         if (opt_showdisassemble) {
3149                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3150                                         }
3151
3152                                         s1 = 0;
3153                                         s2 = 0;
3154                                         d = um->methodref->parseddesc.md->returntype.type;
3155
3156                                 } else {
3157                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3158                                                 sizeof(methodptr) * lm->class->index;
3159
3160                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3161
3162                                         d = lm->parseddesc->returntype.type;
3163                                 }
3164
3165                                 M_ALD(REG_METHODPTR, rd->argintregs[0],
3166                                           OFFSET(java_objectheader, vftbl));
3167                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
3168                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
3169                                 M_CALL(REG_ITMP3);
3170                                 break;
3171                         }
3172
3173                         /* generate method profiling code */
3174
3175                         PROFILE_CYCLE_START;
3176
3177                         /* d contains return type */
3178
3179                         if (d != TYPE_VOID) {
3180                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3181                                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_RESULT);
3182                                         M_INTMOVE(REG_RESULT, s1);
3183                                         emit_store(jd, iptr, iptr->dst, s1);
3184                                 } else {
3185                                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FRESULT);
3186                                         M_FLTMOVE(REG_FRESULT, s1);
3187                                         emit_store(jd, iptr, iptr->dst, s1);
3188                                 }
3189                         }
3190                         break;
3191
3192
3193                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3194
3195                                       /* op1:   0 == array, 1 == class                */
3196                                       /* val.a: (classinfo *) superclass              */
3197
3198                         /*  superclass is an interface:
3199                          *
3200                          *  OK if ((sub == NULL) ||
3201                          *         (sub->vftbl->interfacetablelength > super->index) &&
3202                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3203                          *
3204                          *  superclass is a class:
3205                          *
3206                          *  OK if ((sub == NULL) || (0
3207                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3208                          *         super->vftbl->diffval));
3209                          */
3210
3211                         if (iptr->op1 == 1) {
3212                                 /* object type cast-check */
3213
3214                                 classinfo *super;
3215                                 vftbl_t   *supervftbl;
3216                                 s4         superindex;
3217
3218                                 super = (classinfo *) iptr->val.a;
3219
3220                                 if (super == NULL) {
3221                                         superindex = 0;
3222                                         supervftbl = NULL;
3223                                 }
3224                                 else {
3225                                         superindex = super->index;
3226                                         supervftbl = super->vftbl;
3227                                 }
3228
3229 #if defined(ENABLE_THREADS)
3230                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3231 #endif
3232                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3233
3234                                 /* calculate interface checkcast code size */
3235
3236                                 s2 = 3; /* mov_membase_reg */
3237                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3238
3239                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3240                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3241                                         3 /* test */ + 6 /* jcc */;
3242
3243                                 if (super == NULL)
3244                                         s2 += (opt_showdisassemble ? 5 : 0);
3245
3246                                 /* calculate class checkcast code size */
3247
3248                                 s3 = 3; /* mov_membase_reg */
3249                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3250                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3251
3252 #if 0
3253                                 if (s1 != REG_ITMP1) {
3254                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3255                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3256                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3257                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3258                                         a += 3;    /* sub */
3259
3260                                 } else
3261 #endif
3262                                         {
3263                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3264                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3265                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3266                                         }
3267
3268                                 s3 += 3 /* cmp */ + 6 /* jcc */;
3269
3270                                 if (super == NULL)
3271                                         s3 += (opt_showdisassemble ? 5 : 0);
3272
3273                                 /* if class is not resolved, check which code to call */
3274
3275                                 if (super == NULL) {
3276                                         M_TEST(s1);
3277                                         M_BEQ(6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3278
3279                                         codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3280                                                                                 (constant_classref *) iptr->target, 0);
3281
3282                                         if (opt_showdisassemble) {
3283                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3284                                         }
3285
3286                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
3287                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
3288                                         M_BEQ(s2 + 5);
3289                                 }
3290
3291                                 /* interface checkcast code */
3292
3293                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
3294                                         if (super != NULL) {
3295                                                 M_TEST(s1);
3296                                                 M_BEQ(s2);
3297                                         }
3298
3299                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3300
3301                                         if (super == NULL) {
3302                                                 codegen_addpatchref(cd,
3303                                                                                         PATCHER_checkcast_instanceof_interface,
3304                                                                                         (constant_classref *) iptr->target,
3305                                                                                         0);
3306
3307                                                 if (opt_showdisassemble) {
3308                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3309                                                 }
3310                                         }
3311
3312                                         emit_movl_membase32_reg(cd, REG_ITMP2,
3313                                                                                           OFFSET(vftbl_t, interfacetablelength),
3314                                                                                           REG_ITMP3);
3315                                         /* XXX TWISTI: should this be int arithmetic? */
3316                                         M_LSUB_IMM32(superindex, REG_ITMP3);
3317                                         M_TEST(REG_ITMP3);
3318                                         M_BLE(0);
3319                                         codegen_add_classcastexception_ref(cd, s1);
3320                                         emit_mov_membase32_reg(cd, REG_ITMP2,
3321                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
3322                                                                                          superindex * sizeof(methodptr*),
3323                                                                                          REG_ITMP3);
3324                                         M_TEST(REG_ITMP3);
3325                                         M_BEQ(0);
3326                                         codegen_add_classcastexception_ref(cd, s1);
3327
3328                                         if (super == NULL)
3329                                                 M_JMP_IMM(s3);
3330                                 }
3331
3332                                 /* class checkcast code */
3333
3334                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
3335                                         if (super != NULL) {
3336                                                 M_TEST(s1);
3337                                                 M_BEQ(s3);
3338                                         }
3339
3340                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3341
3342                                         if (super == NULL) {
3343                                                 codegen_addpatchref(cd, PATCHER_checkcast_class,
3344                                                                                         (constant_classref *) iptr->target,
3345                                                                                         0);
3346
3347                                                 if (opt_showdisassemble) {
3348                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3349                                                 }
3350                                         }
3351
3352                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3353 #if defined(ENABLE_THREADS)
3354                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3355 #endif
3356                                         emit_movl_membase32_reg(cd, REG_ITMP2,
3357                                                                                           OFFSET(vftbl_t, baseval),
3358                                                                                           REG_ITMP2);
3359                                         /*                                      if (s1 != REG_ITMP1) { */
3360                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3361                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
3362                                         /*                                                                                              REG_ITMP1); */
3363                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3364                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3365                                         /*                                                                                              REG_ITMP3); */
3366                                         /*  #if defined(ENABLE_THREADS) */
3367                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3368                                         /*  #endif */
3369                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
3370
3371                                         /*                                      } else { */
3372                                         emit_movl_membase32_reg(cd, REG_ITMP3,
3373                                                                                           OFFSET(vftbl_t, baseval),
3374                                                                                           REG_ITMP3);
3375                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3376                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3377                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3378                                         /*                                      } */
3379 #if defined(ENABLE_THREADS)
3380                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3381 #endif
3382                                         M_LCMP(REG_ITMP3, REG_ITMP2);
3383                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3384                                         codegen_add_classcastexception_ref(cd, s1);
3385                                 }
3386
3387                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
3388                         }
3389                         else {
3390                                 /* array type cast-check */
3391
3392                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP2);
3393                                 M_INTMOVE(s1, rd->argintregs[0]);
3394
3395                                 if (iptr->val.a == NULL) {
3396                                         codegen_addpatchref(cd, PATCHER_builtin_arraycheckcast,
3397                                                                                 (constant_classref *) iptr->target, 0);
3398
3399                                         if (opt_showdisassemble) {
3400                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3401                                         }
3402                                 }
3403
3404                                 M_MOV_IMM(iptr->val.a, rd->argintregs[1]);
3405                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
3406                                 M_CALL(REG_ITMP1);
3407
3408                                 /* s1 may have been destroyed over the function call */
3409                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP2);
3410                                 M_TEST(REG_RESULT);
3411                                 M_BEQ(0);
3412                                 codegen_add_classcastexception_ref(cd, s1);
3413
3414                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
3415                         }
3416
3417                         M_INTMOVE(s1, d);
3418                         emit_store(jd, iptr, iptr->dst, d);
3419                         break;
3420
3421                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3422
3423                                       /* op1:   0 == array, 1 == class                */
3424                                       /* val.a: (classinfo *) superclass              */
3425
3426                         /*  superclass is an interface:
3427                          *
3428                          *  return (sub != NULL) &&
3429                          *         (sub->vftbl->interfacetablelength > super->index) &&
3430                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3431                          *
3432                          *  superclass is a class:
3433                          *
3434                          *  return ((sub != NULL) && (0
3435                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3436                          *          super->vftbl->diffvall));
3437                          */
3438
3439                         {
3440                         classinfo *super;
3441                         vftbl_t   *supervftbl;
3442                         s4         superindex;
3443
3444                         super = (classinfo *) iptr->val.a;
3445
3446                         if (!super) {
3447                                 superindex = 0;
3448                                 supervftbl = NULL;
3449
3450                         } else {
3451                                 superindex = super->index;
3452                                 supervftbl = super->vftbl;
3453                         }
3454
3455 #if defined(ENABLE_THREADS)
3456             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3457 #endif
3458
3459                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3460                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
3461                         if (s1 == d) {
3462                                 M_INTMOVE(s1, REG_ITMP1);
3463                                 s1 = REG_ITMP1;
3464                         }
3465
3466                         /* calculate interface instanceof code size */
3467
3468                         s2 = 3; /* mov_membase_reg */
3469                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3470                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3471                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3472                                 3 /* test */ + 4 /* setcc */;
3473
3474                         if (!super)
3475                                 s2 += (opt_showdisassemble ? 5 : 0);
3476
3477                         /* calculate class instanceof code size */
3478
3479                         s3 = 3; /* mov_membase_reg */
3480                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3481                         s3 += 10; /* mov_imm_reg */
3482                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3483                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3484                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3485                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3486                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3487                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3488                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3489
3490                         if (!super)
3491                                 s3 += (opt_showdisassemble ? 5 : 0);
3492
3493                         emit_alu_reg_reg(cd, ALU_XOR, d, d);
3494
3495                         /* if class is not resolved, check which code to call */
3496
3497                         if (!super) {
3498                                 emit_test_reg_reg(cd, s1, s1);
3499                                 emit_jcc(cd, CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3500                                                                                          7 + 6 + s2 + 5 + s3));
3501
3502                                 codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3503                                                                         (constant_classref *) iptr->target, 0);
3504
3505                                 if (opt_showdisassemble) {
3506                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3507                                 }
3508
3509                                 emit_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3510                                 emit_alul_imm_reg(cd, ALU_AND, ACC_INTERFACE, REG_ITMP3);
3511                                 emit_jcc(cd, CC_Z, s2 + 5);
3512                         }
3513
3514                         /* interface instanceof code */
3515
3516                         if (!super || (super->flags & ACC_INTERFACE)) {
3517                                 if (super) {
3518                                         emit_test_reg_reg(cd, s1, s1);
3519                                         emit_jcc(cd, CC_Z, s2);
3520                                 }
3521
3522                                 emit_mov_membase_reg(cd, s1,
3523                                                                            OFFSET(java_objectheader, vftbl),
3524                                                                            REG_ITMP1);
3525                                 if (!super) {
3526                                         codegen_addpatchref(cd,
3527                                                                                 PATCHER_checkcast_instanceof_interface,
3528                                                                                 (constant_classref *) iptr->target, 0);
3529
3530                                         if (opt_showdisassemble) {
3531                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3532                                         }
3533                                 }
3534
3535                                 emit_movl_membase32_reg(cd, REG_ITMP1,
3536                                                                                   OFFSET(vftbl_t, interfacetablelength),
3537                                                                                   REG_ITMP3);
3538                                 emit_alu_imm32_reg(cd, ALU_SUB, superindex, REG_ITMP3);
3539                                 emit_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3540
3541                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3542
3543                                 emit_jcc(cd, CC_LE, a);
3544                                 emit_mov_membase32_reg(cd, REG_ITMP1,
3545                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3546                                                                                  superindex * sizeof(methodptr*),
3547                                                                                  REG_ITMP1);
3548                                 emit_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3549                                 emit_setcc_reg(cd, CC_NE, d);
3550
3551                                 if (!super)
3552                                         emit_jmp_imm(cd, s3);
3553                         }
3554
3555                         /* class instanceof code */
3556
3557                         if (!super || !(super->flags & ACC_INTERFACE)) {
3558                                 if (super) {
3559                                         emit_test_reg_reg(cd, s1, s1);
3560                                         emit_jcc(cd, CC_E, s3);
3561                                 }
3562
3563                                 emit_mov_membase_reg(cd, s1,
3564                                                                            OFFSET(java_objectheader, vftbl),
3565                                                                            REG_ITMP1);
3566
3567                                 if (!super) {
3568                                         codegen_addpatchref(cd, PATCHER_instanceof_class,
3569                                                                                 (constant_classref *) iptr->target, 0);
3570
3571                                         if (opt_showdisassemble) {
3572                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3573                                         }
3574                                 }
3575
3576                                 emit_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3577 #if defined(ENABLE_THREADS)
3578                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3579 #endif
3580                                 emit_movl_membase_reg(cd, REG_ITMP1,
3581                                                                                 OFFSET(vftbl_t, baseval),
3582                                                                                 REG_ITMP1);
3583                                 emit_movl_membase_reg(cd, REG_ITMP2,
3584                                                                                 OFFSET(vftbl_t, diffval),
3585                                                                                 REG_ITMP3);
3586                                 emit_movl_membase_reg(cd, REG_ITMP2,
3587                                                                                 OFFSET(vftbl_t, baseval),
3588                                                                                 REG_ITMP2);
3589 #if defined(ENABLE_THREADS)
3590                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3591 #endif
3592                                 emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
3593                                 emit_alu_reg_reg(cd, ALU_XOR, d, d); /* may be REG_ITMP2 */
3594                                 emit_alu_reg_reg(cd, ALU_CMP, REG_ITMP3, REG_ITMP1);
3595                                 emit_setcc_reg(cd, CC_BE, d);
3596                         }
3597                         emit_store(jd, iptr, iptr->dst, d);
3598                         }
3599                         break;
3600
3601                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3602                                          /* op1 = dimension, val.a = class            */
3603
3604                         /* check for negative sizes and copy sizes to stack if necessary  */
3605
3606                         MCODECHECK((10 * 4 * iptr->op1) + 5 + 10 * 8);
3607
3608                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3609                                 /* copy SAVEDVAR sizes to stack */
3610
3611                                 if (src->varkind != ARGVAR) {
3612                                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP1);
3613                                         M_LST(s2, REG_SP, s1 * 8);
3614                                 }
3615                         }
3616
3617                         /* is a patcher function set? */
3618
3619                         if (iptr->val.a == NULL) {
3620                                 codegen_addpatchref(cd, PATCHER_builtin_multianewarray,
3621                                                                         (constant_classref *) iptr->target, 0);
3622
3623                                 if (opt_showdisassemble) {
3624                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3625                                 }
3626
3627                                 a = 0;
3628
3629                         } else {
3630                                 a = (ptrint) iptr->val.a;
3631                         }
3632
3633                         /* a0 = dimension count */
3634
3635                         M_MOV_IMM(iptr->op1, rd->argintregs[0]);
3636
3637                         /* a1 = arrayvftbl */
3638
3639                         M_MOV_IMM(iptr->val.a, rd->argintregs[1]);
3640
3641                         /* a2 = pointer to dimensions = stack pointer */
3642
3643                         M_MOV(REG_SP, rd->argintregs[2]);
3644
3645                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3646                         M_CALL(REG_ITMP1);
3647
3648                         /* check for exception before result assignment */
3649
3650                         M_TEST(REG_RESULT);
3651                         M_BEQ(0);
3652                         codegen_add_fillinstacktrace_ref(cd);
3653
3654                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_RESULT);
3655                         M_INTMOVE(REG_RESULT, s1);
3656                         emit_store(jd, iptr, iptr->dst, s1);
3657                         break;
3658
3659                 default:
3660                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3661                         return false;
3662         } /* switch */
3663
3664         } /* for instruction */
3665
3666         /* copy values to interface registers */
3667
3668         src = bptr->outstack;
3669         len = bptr->outdepth;
3670         MCODECHECK(512);
3671 #if defined(ENABLE_LSRA)
3672         if (!opt_lsra)
3673 #endif
3674         while (src) {
3675                 len--;
3676                 if ((src->varkind != STACKVAR)) {
3677                         s2 = src->type;
3678                         if (IS_FLT_DBL_TYPE(s2)) {
3679                                 s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
3680                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3681                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3682
3683                                 } else {
3684                                         emit_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3685                                 }
3686
3687                         } else {
3688                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3689                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3690                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3691
3692                                 } else {
3693                                         emit_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3694                                 }
3695                         }
3696                 }
3697                 src = src->prev;
3698         }
3699
3700         /* At the end of a basic block we may have to append some nops,
3701            because the patcher stub calling code might be longer than the
3702            actual instruction. So codepatching does not change the
3703            following block unintentionally. */
3704
3705         if (cd->mcodeptr < cd->lastmcodeptr) {
3706                 while (cd->mcodeptr < cd->lastmcodeptr) {
3707                         M_NOP;
3708                 }
3709         }
3710
3711         } /* if (bptr -> flags >= BBREACHED) */
3712         } /* for basic block */
3713
3714         dseg_createlinenumbertable(cd);
3715
3716
3717         /* generate exception and patcher stubs */
3718
3719         {
3720                 exceptionref *eref;
3721                 patchref     *pref;
3722                 ptrint        mcode;
3723                 u1           *savedmcodeptr;
3724                 u1           *tmpmcodeptr;
3725
3726                 savedmcodeptr = NULL;
3727
3728                 /* generate exception stubs */
3729
3730                 for (eref = cd->exceptionrefs; eref != NULL; eref = eref->next) {
3731                         gen_resolvebranch(cd->mcodebase + eref->branchpos,
3732                                                           eref->branchpos,
3733                                                           cd->mcodeptr - cd->mcodebase);
3734
3735                         MCODECHECK(512);
3736
3737                         /* Check if the exception is an
3738                            ArrayIndexOutOfBoundsException.  If so, move index register
3739                            into a4. */
3740
3741                         if (eref->reg != -1)
3742                                 M_MOV(eref->reg, rd->argintregs[4]);
3743
3744                         /* calcuate exception address */
3745
3746                         M_MOV_IMM(0, rd->argintregs[3]);
3747                         dseg_adddata(cd);
3748                         M_AADD_IMM32(eref->branchpos - 6, rd->argintregs[3]);
3749
3750                         /* move function to call into REG_ITMP3 */
3751
3752                         M_MOV_IMM(eref->function, REG_ITMP3);
3753
3754                         if (savedmcodeptr != NULL) {
3755                                 M_JMP_IMM(savedmcodeptr - cd->mcodeptr - 5);
3756
3757                         } else {
3758                                 savedmcodeptr = cd->mcodeptr;
3759
3760                                 emit_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3761                                 M_MOV(REG_SP, rd->argintregs[1]);
3762                                 M_ALD(rd->argintregs[2], REG_SP, stackframesize * 8);
3763
3764                                 M_ASUB_IMM(2 * 8, REG_SP);
3765                                 M_AST(rd->argintregs[3], REG_SP, 0 * 8);         /* store XPC */
3766
3767                                 M_CALL(REG_ITMP3);
3768
3769                                 M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3770                                 M_AADD_IMM(2 * 8, REG_SP);
3771
3772                                 M_MOV_IMM(asm_handle_exception, REG_ITMP3);
3773                                 M_JMP(REG_ITMP3);
3774                         }
3775                 }
3776
3777
3778                 /* generate code patching stub call code */
3779
3780                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
3781                         /* check size of code segment */
3782
3783                         MCODECHECK(512);
3784
3785                         /* Get machine code which is patched back in later. A
3786                            `call rel32' is 5 bytes long (but read 8 bytes). */
3787
3788                         savedmcodeptr = cd->mcodebase + pref->branchpos;
3789                         mcode = *((ptrint *) savedmcodeptr);
3790
3791                         /* patch in `call rel32' to call the following code */
3792
3793                         tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr          */
3794                         cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position */
3795
3796                         M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
3797
3798                         cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr   */
3799
3800                         /* move pointer to java_objectheader onto stack */
3801
3802 #if defined(ENABLE_THREADS)
3803                         /* create a virtual java_objectheader */
3804
3805                         (void) dseg_addaddress(cd, NULL);                         /* flcword    */
3806                         (void) dseg_addaddress(cd, lock_get_initial_lock_word()); /* monitorPtr */
3807                         a = dseg_addaddress(cd, NULL);                            /* vftbl      */
3808
3809                         emit_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP3);
3810                         M_PUSH(REG_ITMP3);
3811 #else
3812                         M_PUSH_IMM(0);
3813 #endif
3814
3815                         /* move machine code bytes and classinfo pointer into registers */
3816
3817                         M_MOV_IMM(mcode, REG_ITMP3);
3818                         M_PUSH(REG_ITMP3);
3819                         M_MOV_IMM(pref->ref, REG_ITMP3);
3820                         M_PUSH(REG_ITMP3);
3821                         M_MOV_IMM(pref->disp, REG_ITMP3);
3822                         M_PUSH(REG_ITMP3);
3823
3824                         M_MOV_IMM(pref->patcher, REG_ITMP3);
3825                         M_PUSH(REG_ITMP3);
3826
3827                         M_MOV_IMM(asm_patcher_wrapper, REG_ITMP3);
3828                         M_JMP(REG_ITMP3);
3829                 }
3830         }
3831
3832         /* generate replacement-out stubs */
3833
3834         {
3835                 int i;
3836
3837                 replacementpoint = jd->code->rplpoints;
3838
3839                 for (i = 0; i < jd->code->rplpointcount; ++i, ++replacementpoint) {
3840                         /* check code segment size */
3841
3842                         MCODECHECK(512);
3843
3844                         /* note start of stub code */
3845
3846                         replacementpoint->outcode = (u1*) (ptrint)(cd->mcodeptr - cd->mcodebase);
3847
3848                         /* make machine code for patching */
3849
3850                         disp = (ptrint)(replacementpoint->outcode - replacementpoint->pc) - 5;
3851                         replacementpoint->mcode = 0xe9 | ((u8)disp << 8);
3852
3853                         /* push address of `rplpoint` struct */
3854
3855                         M_MOV_IMM(replacementpoint, REG_ITMP3);
3856                         M_PUSH(REG_ITMP3);
3857
3858                         /* jump to replacement function */
3859
3860                         M_MOV_IMM(asm_replacement_out, REG_ITMP3);
3861                         M_JMP(REG_ITMP3);
3862                 }
3863         }
3864
3865         codegen_finish(jd);
3866
3867         /* everything's ok */
3868
3869         return true;
3870 }
3871
3872
3873 /* createcompilerstub **********************************************************
3874
3875    Creates a stub routine which calls the compiler.
3876
3877 *******************************************************************************/
3878
3879 #define COMPILERSTUB_DATASIZE    3 * SIZEOF_VOID_P
3880 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
3881
3882 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
3883
3884
3885 u1 *createcompilerstub(methodinfo *m)
3886 {
3887         u1          *s;                     /* memory to hold the stub            */
3888         ptrint      *d;
3889         codeinfo    *code;
3890         codegendata *cd;
3891         s4           dumpsize;
3892
3893         s = CNEW(u1, COMPILERSTUB_SIZE);
3894
3895         /* set data pointer and code pointer */
3896
3897         d = (ptrint *) s;
3898         s = s + COMPILERSTUB_DATASIZE;
3899
3900         /* mark start of dump memory area */
3901
3902         dumpsize = dump_size();
3903
3904         cd = DNEW(codegendata);
3905         cd->mcodeptr = s;
3906
3907         /* Store the codeinfo pointer in the same place as in the
3908            methodheader for compiled methods. */
3909
3910         code = code_codeinfo_new(m);
3911
3912         d[0] = (ptrint) asm_call_jit_compiler;
3913         d[1] = (ptrint) m;
3914         d[2] = (ptrint) code;
3915
3916         /* code for the stub */
3917
3918         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
3919         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
3920         M_JMP(REG_ITMP3);
3921
3922 #if defined(ENABLE_STATISTICS)
3923         if (opt_stat)
3924                 count_cstub_len += COMPILERSTUB_SIZE;
3925 #endif
3926
3927         /* release dump area */
3928
3929         dump_release(dumpsize);
3930
3931         return s;
3932 }
3933
3934
3935 /* createnativestub ************************************************************
3936
3937    Creates a stub routine which calls a native method.
3938
3939 *******************************************************************************/
3940
3941 u1 *createnativestub(functionptr f, jitdata *jd, methoddesc *nmd)
3942 {
3943         methodinfo   *m;
3944         codeinfo     *code;
3945         codegendata  *cd;
3946         registerdata *rd;
3947         methoddesc   *md;
3948         s4            stackframesize;       /* size of stackframe if needed       */
3949         s4            nativeparams;
3950         s4            i, j;                 /* count variables                    */
3951         s4            t;
3952         s4            s1, s2;
3953
3954         /* get required compiler data */
3955
3956         m    = jd->m;
3957         code = jd->code;
3958         cd   = jd->cd;
3959         rd   = jd->rd;
3960
3961         /* initialize variables */
3962
3963         md = m->parseddesc;
3964         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
3965
3966         /* calculate stack frame size */
3967
3968         stackframesize =
3969                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
3970                 sizeof(localref_table) / SIZEOF_VOID_P +
3971                 INT_ARG_CNT + FLT_ARG_CNT + 1 +         /* + 1 for function address   */
3972                 nmd->memuse;
3973
3974         if (!(stackframesize & 0x1))                /* keep stack 16-byte aligned */
3975                 stackframesize++;
3976
3977         /* create method header */
3978
3979         (void) dseg_addaddress(cd, code);                      /* CodeinfoPointer */
3980         (void) dseg_adds4(cd, stackframesize * 8);             /* FrameSize       */
3981         (void) dseg_adds4(cd, 0);                              /* IsSync          */
3982         (void) dseg_adds4(cd, 0);                              /* IsLeaf          */
3983         (void) dseg_adds4(cd, 0);                              /* IntSave         */
3984         (void) dseg_adds4(cd, 0);                              /* FltSave         */
3985         (void) dseg_addlinenumbertablesize(cd);
3986         (void) dseg_adds4(cd, 0);                              /* ExTableSize     */
3987
3988         /* generate native method profiling code */
3989
3990         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3991                 /* count frequency */
3992
3993                 M_MOV_IMM(code, REG_ITMP3);
3994                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3995         }
3996
3997         /* generate stub code */
3998
3999         M_ASUB_IMM(stackframesize * 8, REG_SP);
4000
4001 #if !defined(NDEBUG)
4002         /* generate call trace */
4003
4004         if (opt_verbosecall) {
4005                 /* save integer and float argument registers */
4006
4007                 for (i = 0, j = 1; i < md->paramcount; i++) {
4008                         if (!md->params[i].inmemory) {
4009                                 s1 = md->params[i].regoff;
4010
4011                                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4012                                         M_LST(rd->argintregs[s1], REG_SP, j * 8);
4013                                 else
4014                                         M_DST(rd->argfltregs[s1], REG_SP, j * 8);
4015
4016                                 j++;
4017                         }
4018                 }
4019
4020                 /* show integer hex code for float arguments */
4021
4022                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++) {
4023                         /* if the paramtype is a float, we have to right shift all
4024                            following integer registers */
4025
4026                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type)) {
4027                                 for (s1 = INT_ARG_CNT - 2; s1 >= i; s1--)
4028                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
4029
4030                                 emit_movd_freg_reg(cd, rd->argfltregs[j], rd->argintregs[i]);
4031                                 j++;
4032                         }
4033                 }
4034
4035                 M_MOV_IMM(m, REG_ITMP1);
4036                 M_AST(REG_ITMP1, REG_SP, 0 * 8);
4037                 M_MOV_IMM(builtin_trace_args, REG_ITMP1);
4038                 M_CALL(REG_ITMP1);
4039
4040                 /* restore integer and float argument registers */
4041
4042                 for (i = 0, j = 1; i < md->paramcount; i++) {
4043                         if (!md->params[i].inmemory) {
4044                                 s1 = md->params[i].regoff;
4045
4046                                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4047                                         M_LLD(rd->argintregs[s1], REG_SP, j * 8);
4048                                 else
4049                                         M_DLD(rd->argfltregs[s1], REG_SP, j * 8);
4050
4051                                 j++;
4052                         }
4053                 }
4054         }
4055 #endif /* !defined(NDEBUG) */
4056
4057         /* get function address (this must happen before the stackframeinfo) */
4058
4059 #if !defined(WITH_STATIC_CLASSPATH)
4060         if (f == NULL) {
4061                 codegen_addpatchref(cd, PATCHER_resolve_native, m, 0);
4062
4063                 if (opt_showdisassemble) {
4064                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
4065                 }
4066         }
4067 #endif
4068
4069         M_MOV_IMM(f, REG_ITMP3);
4070
4071
4072         /* save integer and float argument registers */
4073
4074         for (i = 0, j = 0; i < md->paramcount; i++) {
4075                 if (!md->params[i].inmemory) {
4076                         s1 = md->params[i].regoff;
4077
4078                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4079                                 M_LST(rd->argintregs[s1], REG_SP, j * 8);
4080                         else
4081                                 M_DST(rd->argfltregs[s1], REG_SP, j * 8);
4082
4083                         j++;
4084                 }
4085         }
4086
4087         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4088
4089         /* create dynamic stack info */
4090
4091         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4092         emit_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[1]);
4093         M_ALEA(REG_SP, stackframesize * 8 + SIZEOF_VOID_P, rd->argintregs[2]);
4094         M_ALD(rd->argintregs[3], REG_SP, stackframesize * 8);
4095         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
4096         M_CALL(REG_ITMP1);
4097
4098         /* restore integer and float argument registers */
4099
4100         for (i = 0, j = 0; i < md->paramcount; i++) {
4101                 if (!md->params[i].inmemory) {
4102                         s1 = md->params[i].regoff;
4103
4104                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4105                                 M_LLD(rd->argintregs[s1], REG_SP, j * 8);
4106                         else
4107                                 M_DLD(rd->argfltregs[s1], REG_SP, j * 8);
4108
4109                         j++;
4110                 }
4111         }
4112
4113         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4114
4115
4116         /* copy or spill arguments to new locations */
4117
4118         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
4119                 t = md->paramtypes[i].type;
4120
4121                 if (IS_INT_LNG_TYPE(t)) {
4122                         if (!md->params[i].inmemory) {
4123                                 s1 = rd->argintregs[md->params[i].regoff];
4124
4125                                 if (!nmd->params[j].inmemory) {
4126                                         s2 = rd->argintregs[nmd->params[j].regoff];
4127                                         M_INTMOVE(s1, s2);
4128
4129                                 } else {
4130                                         s2 = nmd->params[j].regoff;
4131                                         M_LST(s1, REG_SP, s2 * 8);
4132                                 }
4133
4134                         } else {
4135                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4136                                 s2 = nmd->params[j].regoff;
4137                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
4138                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
4139                         }
4140
4141                 } else {
4142                         /* We only copy spilled float arguments, as the float argument    */
4143                         /* registers keep unchanged.                                      */
4144
4145                         if (md->params[i].inmemory) {
4146                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4147                                 s2 = nmd->params[j].regoff;
4148
4149                                 if (IS_2_WORD_TYPE(t)) {
4150                                         M_DLD(REG_FTMP1, REG_SP, s1 * 8);
4151                                         M_DST(REG_FTMP1, REG_SP, s2 * 8);
4152                                 } else {
4153                                         M_FLD(REG_FTMP1, REG_SP, s1 * 8);
4154                                         M_FST(REG_FTMP1, REG_SP, s2 * 8);
4155                                 }
4156                         }
4157                 }
4158         }
4159
4160         /* put class into second argument register */
4161
4162         if (m->flags & ACC_STATIC)
4163                 M_MOV_IMM(m->class, rd->argintregs[1]);
4164
4165         /* put env into first argument register */
4166
4167         M_MOV_IMM(_Jv_env, rd->argintregs[0]);
4168
4169         /* do the native function call */
4170
4171         M_CALL(REG_ITMP3);
4172
4173         /* save return value */
4174
4175         if (md->returntype.type != TYPE_VOID) {
4176                 if (IS_INT_LNG_TYPE(md->returntype.type))
4177                         M_LST(REG_RESULT, REG_SP, 0 * 8);
4178                 else
4179                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
4180         }
4181
4182 #if !defined(NDEBUG)
4183         /* generate call trace */
4184
4185         if (opt_verbosecall) {
4186                 /* just restore the value we need, don't care about the other */
4187
4188                 if (md->returntype.type != TYPE_VOID) {
4189                         if (IS_INT_LNG_TYPE(md->returntype.type))
4190                                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
4191                         else
4192                                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4193                 }
4194
4195                 M_MOV_IMM(m, rd->argintregs[0]);
4196                 M_MOV(REG_RESULT, rd->argintregs[1]);
4197                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4198                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4199
4200                 M_MOV_IMM(builtin_displaymethodstop, REG_ITMP1);
4201                 M_CALL(REG_ITMP1);
4202         }
4203 #endif /* !defined(NDEBUG) */
4204
4205         /* remove native stackframe info */
4206
4207         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4208         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
4209         M_CALL(REG_ITMP1);
4210         M_MOV(REG_RESULT, REG_ITMP3);
4211
4212         /* restore return value */
4213
4214         if (md->returntype.type != TYPE_VOID) {
4215                 if (IS_INT_LNG_TYPE(md->returntype.type))
4216                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
4217                 else
4218                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4219         }
4220
4221         /* remove stackframe */
4222
4223         M_AADD_IMM(stackframesize * 8, REG_SP);
4224
4225         /* test for exception */
4226
4227         M_TEST(REG_ITMP3);
4228         M_BNE(1);
4229         M_RET;
4230
4231         /* handle exception */
4232
4233         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
4234         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
4235         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
4236
4237         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
4238         M_JMP(REG_ITMP3);
4239
4240
4241         /* process patcher calls **************************************************/
4242
4243         {
4244                 patchref *pref;
4245                 ptrint    mcode;
4246                 u1       *savedmcodeptr;
4247                 u1       *tmpmcodeptr;
4248 #if defined(ENABLE_THREADS)
4249                 s4        disp;
4250 #endif
4251
4252                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4253                         /* Get machine code which is patched back in later. A
4254                            `call rel32' is 5 bytes long (but read 8 bytes). */
4255
4256                         savedmcodeptr = cd->mcodebase + pref->branchpos;
4257                         mcode = *((ptrint *) savedmcodeptr);
4258
4259                         /* patch in `call rel32' to call the following code               */
4260
4261                         tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr          */
4262                         cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position */
4263
4264                         M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
4265
4266                         cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr   */
4267
4268                         /* move pointer to java_objectheader onto stack */
4269
4270 #if defined(ENABLE_THREADS)
4271                         /* create a virtual java_objectheader */
4272
4273                         (void) dseg_addaddress(cd, NULL);                         /* flcword    */
4274                         (void) dseg_addaddress(cd, lock_get_initial_lock_word()); /* monitorPtr */
4275                         disp = dseg_addaddress(cd, NULL);                         /* vftbl      */
4276
4277                         emit_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP3);
4278                         M_PUSH(REG_ITMP3);
4279 #else
4280                         M_PUSH_IMM(0);
4281 #endif
4282
4283                         /* move machine code bytes and classinfo pointer into registers */
4284
4285                         M_MOV_IMM(mcode, REG_ITMP3);
4286                         M_PUSH(REG_ITMP3);
4287                         M_MOV_IMM(pref->ref, REG_ITMP3);
4288                         M_PUSH(REG_ITMP3);
4289                         M_MOV_IMM(pref->disp, REG_ITMP3);
4290                         M_PUSH(REG_ITMP3);
4291
4292                         M_MOV_IMM(pref->patcher, REG_ITMP3);
4293                         M_PUSH(REG_ITMP3);
4294
4295                         M_MOV_IMM(asm_patcher_wrapper, REG_ITMP3);
4296                         M_JMP(REG_ITMP3);
4297                 }
4298         }
4299
4300         codegen_finish(jd);
4301
4302         return code->entrypoint;
4303 }
4304
4305
4306 /*
4307  * These are local overrides for various environment variables in Emacs.
4308  * Please do not remove this and leave it at the end of the file, where
4309  * Emacs will automagically detect them.
4310  * ---------------------------------------------------------------------
4311  * Local variables:
4312  * mode: c
4313  * indent-tabs-mode: t
4314  * c-basic-offset: 4
4315  * tab-width: 4
4316  * End:
4317  * vim:noexpandtab:sw=4:ts=4:
4318  */