src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31             Edwin Steiner
  32
  33    $Id: codegen.c 4937 2006-05-18 14:33:32Z edwin $
  34
  35 */
  36
  37
  38 #include "config.h"
  39
  40 #include <assert.h>
  41 #include <stdio.h>
  42
  43 #include "vm/types.h"
  44
  45 #include "md.h"
  46 #include "md-abi.h"
  47
  48 #include "vm/jit/x86_64/arch.h"
  49 #include "vm/jit/x86_64/codegen.h"
  50 #include "vm/jit/x86_64/md-emit.h"
  51
  52 #include "mm/memory.h"
  53 #include "native/jni.h"
  54 #include "native/native.h"
  55 #include "vm/builtin.h"
  56 #include "vm/exceptions.h"
  57 #include "vm/global.h"
  58 #include "vm/loader.h"
  59 #include "vm/options.h"
  60 #include "vm/statistics.h"
  61 #include "vm/stringlocal.h"
  62 #include "vm/vm.h"
  63 #include "vm/jit/asmpart.h"
  64 #include "vm/jit/codegen-common.h"
  65 #include "vm/jit/dseg.h"
  66 #include "vm/jit/emit.h"
  67 #include "vm/jit/jit.h"
  68 #include "vm/jit/methodheader.h"
  69 #include "vm/jit/parse.h"
  70 #include "vm/jit/patcher.h"
  71 #include "vm/jit/reg.h"
  72 #include "vm/jit/replace.h"
  73
  74 #if defined(ENABLE_LSRA)
  75 # include "vm/jit/allocator/lsra.h"
  76 #endif
  77
  78
  79 /* codegen *********************************************************************
  80
  81    Generates machine code.
  82
  83 *******************************************************************************/
  84
  85 bool codegen(jitdata *jd)
  86 {
  87         methodinfo         *m;
  88         codegendata        *cd;
  89         registerdata       *rd;
  90         s4                  len, s1, s2, s3, d, disp;
  91         u2                  currentline;
  92         ptrint              a;
  93         s4                  stackframesize;
  94         stackptr            src;
  95         varinfo            *var;
  96         basicblock         *bptr;
  97         instruction        *iptr;
  98         exceptiontable     *ex;
  99         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 100         builtintable_entry *bte;
 101         methoddesc         *md;
 102         rplpoint           *replacementpoint;
 103
 104         /* get required compiler data */
 105
 106         m  = jd->m;
 107         cd = jd->cd;
 108         rd = jd->rd;
 109
 110         /* prevent compiler warnings */
 111
 112         d = 0;
 113         lm = NULL;
 114         bte = NULL;
 115
 116         {
 117         s4 i, p, t, l;
 118         s4 savedregs_num;
 119
 120         savedregs_num = 0;
 121
 122         /* space to save used callee saved registers */
 123
 124         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 125         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 126
 127         stackframesize = rd->memuse + savedregs_num;
 128
 129 #if defined(ENABLE_THREADS)
 130         /* space to save argument of monitor_enter */
 131
 132         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 133                 stackframesize++;
 134 #endif
 135
 136     /* Keep stack of non-leaf functions 16-byte aligned for calls into native */
 137         /* code e.g. libc or jni (alignment problems with movaps).                */
 138
 139         if (!m->isleafmethod || opt_verbosecall)
 140                 stackframesize |= 0x1;
 141
 142         /* create method header */
 143
 144         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 145         (void) dseg_adds4(cd, stackframesize * 8);              /* FrameSize      */
 146
 147 #if defined(ENABLE_THREADS)
 148         /* IsSync contains the offset relative to the stack pointer for the
 149            argument of monitor_exit used in the exception handler. Since the
 150            offset could be zero and give a wrong meaning of the flag it is
 151            offset by one.
 152         */
 153
 154         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 155                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);        /* IsSync         */
 156         else
 157 #endif
 158                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 159
 160         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 161         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse);  /* IntSave        */
 162         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse);  /* FltSave        */
 163
 164         (void) dseg_addlinenumbertablesize(cd);
 165
 166         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 167
 168         /* create exception table */
 169
 170         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 171                 dseg_addtarget(cd, ex->start);
 172                 dseg_addtarget(cd, ex->end);
 173                 dseg_addtarget(cd, ex->handler);
 174                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 175         }
 176
 177         /* generate method profiling code */
 178
 179         if (opt_prof) {
 180                 /* count frequency */
 181
 182                 M_MOV_IMM(m, REG_ITMP3);
 183                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(methodinfo, frequency));
 184
 185                 PROFILE_CYCLE_START;
 186         }
 187
 188         /* create stack frame (if necessary) */
 189
 190         if (stackframesize)
 191                 M_ASUB_IMM(stackframesize * 8, REG_SP);
 192
 193         /* save used callee saved registers */
 194
 195         p = stackframesize;
 196         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 197                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 198         }
 199         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 200                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 201         }
 202
 203         /* take arguments out of register or stack frame */
 204
 205         md = m->parseddesc;
 206
 207         for (p = 0, l = 0; p < md->paramcount; p++) {
 208                 t = md->paramtypes[p].type;
 209                 var = &(rd->locals[l][t]);
 210                 l++;
 211                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 212                         l++;
 213                 if (var->type < 0)
 214                         continue;
 215                 s1 = md->params[p].regoff;
 216                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 217                         s2 = rd->argintregs[s1];
 218                         if (!md->params[p].inmemory) {           /* register arguments    */
 219                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 220                                         M_INTMOVE(s2, var->regoff);
 221
 222                                 } else {                             /* reg arg -> spilled    */
 223                                     M_LST(s2, REG_SP, var->regoff * 8);
 224                                 }
 225
 226                         } else {                                 /* stack arguments       */
 227                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 228                                         /* + 8 for return address */
 229                                         M_LLD(var->regoff, REG_SP, (stackframesize + s1) * 8 + 8);
 230
 231                                 } else {                             /* stack arg -> spilled  */
 232                                         var->regoff = stackframesize + s1 + 1;
 233                                 }
 234                         }
 235
 236                 } else {                                     /* floating args         */
 237                         if (!md->params[p].inmemory) {           /* register arguments    */
 238                                 s2 = rd->argfltregs[s1];
 239                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 240                                         M_FLTMOVE(s2, var->regoff);
 241
 242                                 } else {                                         /* reg arg -> spilled    */
 243                                         M_DST(s2, REG_SP, var->regoff * 8);
 244                                 }
 245
 246                         } else {                                 /* stack arguments       */
 247                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 248                                         M_DLD(var->regoff, REG_SP, (stackframesize + s1) * 8 + 8);
 249
 250                                 } else {
 251                                         var->regoff = stackframesize + s1 + 1;
 252                                 }
 253                         }
 254                 }
 255         }  /* end for */
 256
 257         /* save monitorenter argument */
 258
 259 #if defined(ENABLE_THREADS)
 260         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 261                 /* stack offset for monitor argument */
 262
 263                 s1 = rd->memuse;
 264
 265                 if (opt_verbosecall) {
 266                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 267
 268                         for (p = 0; p < INT_ARG_CNT; p++)
 269                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 270
 271                         for (p = 0; p < FLT_ARG_CNT; p++)
 272                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 273
 274                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 275                 }
 276
 277                 /* decide which monitor enter function to call */
 278
 279                 if (m->flags & ACC_STATIC) {
 280                         M_MOV_IMM(m->class, REG_ITMP1);
 281                         M_AST(REG_ITMP1, REG_SP, s1 * 8);
 282                         M_INTMOVE(REG_ITMP1, rd->argintregs[0]);
 283                         M_MOV_IMM(BUILTIN_staticmonitorenter, REG_ITMP1);
 284                         M_CALL(REG_ITMP1);
 285
 286                 } else {
 287                         M_TEST(rd->argintregs[0]);
 288                         M_BEQ(0);
 289                         codegen_add_nullpointerexception_ref(cd);
 290                         M_AST(rd->argintregs[0], REG_SP, s1 * 8);
 291                         M_MOV_IMM(BUILTIN_monitorenter, REG_ITMP1);
 292                         M_CALL(REG_ITMP1);
 293                 }
 294
 295                 if (opt_verbosecall) {
 296                         for (p = 0; p < INT_ARG_CNT; p++)
 297                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 298
 299                         for (p = 0; p < FLT_ARG_CNT; p++)
 300                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 301
 302                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 303                 }
 304         }
 305 #endif
 306
 307 #if !defined(NDEBUG)
 308         /* Copy argument registers to stack and call trace function with
 309            pointer to arguments on stack. */
 310
 311         if (opt_verbosecall) {
 312                 M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 313
 314                 /* save integer argument registers */
 315
 316                 for (p = 0; p < INT_ARG_CNT; p++)
 317                         M_LST(rd->argintregs[p], REG_SP, (1 + p) * 8);
 318
 319                 /* save float argument registers */
 320
 321                 for (p = 0; p < FLT_ARG_CNT; p++)
 322                         M_DST(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 323
 324                 /* save temporary registers for leaf methods */
 325
 326                 if (m->isleafmethod) {
 327                         for (p = 0; p < INT_TMP_CNT; p++)
 328                                 M_LST(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 329
 330                         for (p = 0; p < FLT_TMP_CNT; p++)
 331                                 M_DST(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 332                 }
 333
 334                 /* show integer hex code for float arguments */
 335
 336                 for (p = 0, l = 0; p < md->paramcount && p < INT_ARG_CNT; p++) {
 337                         /* If the paramtype is a float, we have to right shift all
 338                            following integer registers. */
 339
 340                         if (IS_FLT_DBL_TYPE(md->paramtypes[p].type)) {
 341                                 for (s1 = INT_ARG_CNT - 2; s1 >= p; s1--)
 342                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
 343
 344                                 emit_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 345                                 l++;
 346                         }
 347                 }
 348
 349                 M_MOV_IMM(m, REG_ITMP2);
 350                 M_AST(REG_ITMP2, REG_SP, 0 * 8);
 351                 M_MOV_IMM(builtin_trace_args, REG_ITMP1);
 352                 M_CALL(REG_ITMP1);
 353
 354                 /* restore integer argument registers */
 355
 356                 for (p = 0; p < INT_ARG_CNT; p++)
 357                         M_LLD(rd->argintregs[p], REG_SP, (1 + p) * 8);
 358
 359                 /* restore float argument registers */
 360
 361                 for (p = 0; p < FLT_ARG_CNT; p++)
 362                         M_DLD(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 363
 364                 /* restore temporary registers for leaf methods */
 365
 366                 if (m->isleafmethod) {
 367                         for (p = 0; p < INT_TMP_CNT; p++)
 368                                 M_LLD(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 369
 370                         for (p = 0; p < FLT_TMP_CNT; p++)
 371                                 M_DLD(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 372                 }
 373
 374                 M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 375         }
 376 #endif /* !defined(NDEBUG) */
 377
 378         }
 379
 380         /* end of header generation */
 381
 382         replacementpoint = jd->code->rplpoints;
 383
 384         /* walk through all basic blocks */
 385
 386         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 387
 388                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 389
 390                 if (bptr->flags >= BBREACHED) {
 391
 392                         /* branch resolving */
 393
 394                         branchref *bref;
 395                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 396                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 397                                                                   bref->branchpos,
 398                                                                   bptr->mpc);
 399                         }
 400
 401                 /* handle replacement points */
 402
 403                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 404                         replacementpoint->pc = (u1*)(ptrint)bptr->mpc; /* will be resolved later */
 405
 406                         replacementpoint++;
 407
 408                         assert(cd->lastmcodeptr <= cd->mcodeptr);
 409                         cd->lastmcodeptr = cd->mcodeptr + 5; /* 5 byte jmp patch */
 410                 }
 411
 412                 /* copy interface registers to their destination */
 413
 414                 src = bptr->instack;
 415                 len = bptr->indepth;
 416                 MCODECHECK(512);
 417
 418                 /* generate basicblock profiling code */
 419
 420                 if (opt_prof_bb) {
 421                         /* count frequency */
 422
 423                         M_MOV_IMM(m->bbfrequency, REG_ITMP2);
 424                         M_IINC_MEMBASE(REG_ITMP2, bptr->debug_nr * 4);
 425
 426                         /* if this is an exception handler, start profiling again */
 427
 428                         if (bptr->type == BBTYPE_EXH)
 429                                 PROFILE_CYCLE_START;
 430                 }
 431
 432 #if defined(ENABLE_LSRA)
 433                 if (opt_lsra) {
 434                         while (src != NULL) {
 435                                 len--;
 436                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 437                                         if (bptr->type == BBTYPE_SBR) {
 438 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 439                                                 if (!(src->flags & INMEMORY))
 440                                                         d = src->regoff;
 441                                                 else
 442                                                         d = REG_ITMP1;
 443                                                 M_POP(d);
 444                                                 emit_store(jd, NULL, src, d);
 445
 446                                         } else if (bptr->type == BBTYPE_EXH) {
 447 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 448                                                 if (!(src->flags & INMEMORY))
 449                                                         d= src->regoff;
 450                                                 else
 451                                                         d=REG_ITMP1;
 452                                                 M_INTMOVE(REG_ITMP1, d);
 453                                                 emit_store(jd, NULL, src, d);
 454                                         }
 455                                 }
 456                                 src = src->prev;
 457                         }
 458
 459                 } else {
 460 #endif
 461
 462                 while (src != NULL) {
 463                         len--;
 464                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 465                                 if (bptr->type == BBTYPE_SBR) {
 466                                         d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 467                                         M_POP(d);
 468                                         emit_store(jd, NULL, src, d);
 469
 470                                 } else if (bptr->type == BBTYPE_EXH) {
 471                                         d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 472                                         M_INTMOVE(REG_ITMP1, d);
 473                                         emit_store(jd, NULL, src, d);
 474                                 }
 475
 476                         } else {
 477                                 d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 478                                 if ((src->varkind != STACKVAR)) {
 479                                         s2 = src->type;
 480                                         if (IS_FLT_DBL_TYPE(s2)) {
 481                                                 s1 = rd->interfaces[len][s2].regoff;
 482
 483                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 484                                                         M_FLTMOVE(s1, d);
 485                                                 else
 486                                                         M_DLD(d, REG_SP, s1 * 8);
 487
 488                                                 emit_store(jd, NULL, src, d);
 489
 490                                         } else {
 491                                                 s1 = rd->interfaces[len][s2].regoff;
 492
 493                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 494                                                         M_INTMOVE(s1, d);
 495                                                 else
 496                                                         M_LLD(d, REG_SP, s1 * 8);
 497
 498                                                 emit_store(jd, NULL, src, d);
 499                                         }
 500                                 }
 501                         }
 502                         src = src->prev;
 503                 }
 504 #if defined(ENABLE_LSRA)
 505                 }
 506 #endif
 507                 /* walk through all instructions */
 508
 509                 src = bptr->instack;
 510                 len = bptr->icount;
 511                 currentline = 0;
 512
 513                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 514                         if (iptr->line != currentline) {
 515                                 dseg_addlinenumber(cd, iptr->line);
 516                                 currentline = iptr->line;
 517                         }
 518
 519                         MCODECHECK(1024);                         /* 1KB should be enough */
 520
 521                         switch (iptr->opc) {
 522                         case ICMD_INLINE_START: /* internal ICMDs                         */
 523                         case ICMD_INLINE_END:
 524                                 break;
 525
 526                         case ICMD_NOP:    /* ...  ==> ...                                 */
 527                                 break;
 528
 529                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 530
 531                                 if (src->flags & INMEMORY)
 532                                         M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
 533                                 else
 534                                         M_TEST(src->regoff);
 535                                 M_BEQ(0);
 536                                 codegen_add_nullpointerexception_ref(cd);
 537                                 break;
 538
 539                 /* constant operations ************************************************/
 540
 541                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 542                                       /* op1 = 0, val.i = constant                    */
 543
 544                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 545                         ICONST(d, iptr->val.i);
 546                         emit_store(jd, iptr, iptr->dst, d);
 547                         break;
 548
 549                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 550                                       /* op1 = 0, val.l = constant                    */
 551
 552                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 553                         LCONST(d, iptr->val.l);
 554                         emit_store(jd, iptr, iptr->dst, d);
 555                         break;
 556
 557                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 558                                       /* op1 = 0, val.f = constant                    */
 559
 560                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 561                         disp = dseg_addfloat(cd, iptr->val.f);
 562                         emit_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + disp, d);
 563                         emit_store(jd, iptr, iptr->dst, d);
 564                         break;
 565
 566                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 567                                       /* op1 = 0, val.d = constant                    */
 568
 569                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 570                         disp = dseg_adddouble(cd, iptr->val.d);
 571                         emit_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, d);
 572                         emit_store(jd, iptr, iptr->dst, d);
 573                         break;
 574
 575                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 576                                       /* op1 = 0, val.a = constant                    */
 577
 578                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 579
 580                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 581 /*                              PROFILE_CYCLE_STOP; */
 582
 583                                 codegen_addpatchref(cd, PATCHER_aconst,
 584                                                                         ICMD_ACONST_UNRESOLVED_CLASSREF(iptr), 0);
 585
 586                                 if (opt_showdisassemble) {
 587                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
 588                                 }
 589
 590 /*                              PROFILE_CYCLE_START; */
 591
 592                                 M_MOV_IMM(NULL, d);
 593
 594                         } else {
 595                                 if (iptr->val.a == 0)
 596                                         M_CLR(d);
 597                                 else
 598                                         M_MOV_IMM(iptr->val.a, d);
 599                         }
 600                         emit_store(jd, iptr, iptr->dst, d);
 601                         break;
 602
 603
 604                 /* load/store operations **********************************************/
 605
 606                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 607                                       /* op1 = local variable                         */
 608
 609                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 610                         if ((iptr->dst->varkind == LOCALVAR) &&
 611                                 (iptr->dst->varnum == iptr->op1))
 612                                 break;
 613                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 614                         if (var->flags & INMEMORY)
 615                                 M_ILD(d, REG_SP, var->regoff * 8);
 616                         else
 617                                 M_INTMOVE(var->regoff, d);
 618                         emit_store(jd, iptr, iptr->dst, d);
 619                         break;
 620
 621                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 622                 case ICMD_ALOAD:      /* op1 = local variable                         */
 623
 624                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 625                         if ((iptr->dst->varkind == LOCALVAR) &&
 626                                 (iptr->dst->varnum == iptr->op1))
 627                                 break;
 628                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 629                         if (var->flags & INMEMORY)
 630                                 M_LLD(d, REG_SP, var->regoff * 8);
 631                         else
 632                                 M_INTMOVE(var->regoff, d);
 633                         emit_store(jd, iptr, iptr->dst, d);
 634                         break;
 635
 636                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 637                 case ICMD_DLOAD:      /* op1 = local variable                         */
 638
 639                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 640                         if ((iptr->dst->varkind == LOCALVAR) &&
 641                             (iptr->dst->varnum == iptr->op1))
 642                                 break;
 643                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 644                         if (var->flags & INMEMORY)
 645                                 M_DLD(d, REG_SP, var->regoff * 8);
 646                         else
 647                                 M_FLTMOVE(var->regoff, d);
 648                         emit_store(jd, iptr, iptr->dst, d);
 649                         break;
 650
 651                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 652                                       /* op1 = local variable                         */
 653
 654                         if ((src->varkind == LOCALVAR) && (src->varnum == iptr->op1))
 655                                 break;
 656                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 657                         if (var->flags & INMEMORY) {
 658                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 659                                 M_IST(s1, REG_SP, var->regoff * 8);
 660                         } else {
 661                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 662                                 M_INTMOVE(s1, var->regoff);
 663                         }
 664                         break;
 665
 666                 case ICMD_LSTORE:     /* ..., value  ==> ...                          */
 667                 case ICMD_ASTORE:     /* op1 = local variable                         */
 668
 669                         if ((src->varkind == LOCALVAR) && (src->varnum == iptr->op1))
 670                                 break;
 671                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 672                         if (var->flags & INMEMORY) {
 673                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 674                                 M_LST(s1, REG_SP, var->regoff * 8);
 675                         } else {
 676                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 677                                 M_INTMOVE(s1, var->regoff);
 678                         }
 679                         break;
 680
 681                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 682                 case ICMD_DSTORE:     /* op1 = local variable                         */
 683
 684                         if ((src->varkind == LOCALVAR) &&
 685                             (src->varnum == iptr->op1)) {
 686                                 break;
 687                         }
 688                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 689                         if (var->flags & INMEMORY) {
 690                                 s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
 691                                 M_DST(s1, REG_SP, var->regoff * 8);
 692                         } else {
 693                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 694                                 M_FLTMOVE(s1, var->regoff);
 695                         }
 696                         break;
 697
 698
 699                 /* pop/dup/swap operations ********************************************/
 700
 701                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 702
 703                 case ICMD_POP:        /* ..., value  ==> ...                          */
 704                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 705                         break;
 706
 707                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 708                         M_COPY(src, iptr->dst);
 709                         break;
 710
 711                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 712
 713                         M_COPY(src,       iptr->dst);
 714                         M_COPY(src->prev, iptr->dst->prev);
 715                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 716                         break;
 717
 718                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 719
 720                         M_COPY(src,             iptr->dst);
 721                         M_COPY(src->prev,       iptr->dst->prev);
 722                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 723                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 724                         break;
 725
 726                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 727
 728                         M_COPY(src,       iptr->dst);
 729                         M_COPY(src->prev, iptr->dst->prev);
 730                         break;
 731
 732                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 733
 734                         M_COPY(src,             iptr->dst);
 735                         M_COPY(src->prev,       iptr->dst->prev);
 736                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 737                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 738                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 739                         break;
 740
 741                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 742
 743                         M_COPY(src,                   iptr->dst);
 744                         M_COPY(src->prev,             iptr->dst->prev);
 745                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 746                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 747                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 748                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 749                         break;
 750
 751                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 752
 753                         M_COPY(src,       iptr->dst->prev);
 754                         M_COPY(src->prev, iptr->dst);
 755                         break;
 756
 757
 758                 /* integer operations *************************************************/
 759
 760 #define RISC_STYLE 1
 761
 762                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 763
 764 #if RISC_STYLE
 765                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 766                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 767                         M_INTMOVE(s1, d);
 768                         M_INEG(d);
 769                         emit_store(jd, iptr, iptr->dst, d);
 770 #else
 771                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 772                         if (iptr->dst->flags & INMEMORY) {
 773                                 if (src->flags & INMEMORY) {
 774                                         if (src->regoff == iptr->dst->regoff)
 775                                                 M_INEG_MEMBASE(REG_SP, iptr->dst->regoff * 8);
 776                                         else {
 777                                                 M_ILD(REG_ITMP1, REG_SP, src->regoff * 8);
 778                                                 M_INEG(REG_ITMP1);
 779                                                 M_IST(REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 780                                         }
 781
 782                                 } else {
 783                                         M_IST(src->regoff, REG_SP, iptr->dst->regoff * 8);
 784                                         M_INEG_MEMBASE(REG_SP, iptr->dst->regoff * 8);
 785                                 }
 786
 787                         } else {
 788                                 if (src->flags & INMEMORY) {
 789                                         M_ILD(iptr->dst->regoff, REG_SP, src->regoff * 8);
 790                                         M_INEG(iptr->dst->regoff);
 791
 792                                 } else {
 793                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 794                                         M_INEG(iptr->dst->regoff);
 795                                 }
 796                         }
 797 #endif
 798                         break;
 799
 800                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 801
 802 #if RISC_STYLE
 803                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 804                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 805                         M_INTMOVE(s1, d);
 806                         M_LNEG(d);
 807                         emit_store(jd, iptr, iptr->dst, d);
 808 #else
 809                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 810                         if (iptr->dst->flags & INMEMORY) {
 811                                 if (src->flags & INMEMORY) {
 812                                         if (src->regoff == iptr->dst->regoff)
 813                                                 M_LNEG_MEMBASE(REG_SP, iptr->dst->regoff * 8);
 814                                         else {
 815                                                 M_LLD(REG_ITMP1, REG_SP, src->regoff * 8);
 816                                                 M_LNEG(REG_ITMP1);
 817                                                 M_LST(REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 818                                         }
 819
 820                                 } else {
 821                                         M_LST(src->regoff, REG_SP, iptr->dst->regoff * 8);
 822                                         M_LNEG_MEMBASE(REG_SP, iptr->dst->regoff * 8);
 823                                 }
 824
 825                         } else {
 826                                 if (src->flags & INMEMORY) {
 827                                         M_LLD(iptr->dst->regoff, REG_SP, src->regoff * 8);
 828                                         M_LNEG(iptr->dst->regoff);
 829
 830                                 } else {
 831                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 832                                         M_LNEG(iptr->dst->regoff);
 833                                 }
 834                         }
 835 #endif
 836                         break;
 837
 838                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 839
 840                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 841                         if (src->flags & INMEMORY)
 842                                 M_ISEXT_MEMBASE(REG_SP, src->regoff * 8, d);
 843                         else
 844                                 M_ISEXT(src->regoff, d);
 845                         emit_store(jd, iptr, iptr->dst, d);
 846                         break;
 847
 848                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 849
 850                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 851                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 852                         M_INTMOVE(s1, d);
 853                         emit_store(jd, iptr, iptr->dst, d);
 854                         break;
 855
 856                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 857
 858                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 859                         if (src->flags & INMEMORY)
 860                                 M_BSEXT_MEMBASE(REG_SP, src->regoff * 8, d);
 861                         else
 862                                 M_BSEXT(src->regoff, d);
 863                         emit_store(jd, iptr, iptr->dst, d);
 864                         break;
 865
 866                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 867
 868                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 869                         if (src->flags & INMEMORY)
 870                                 M_CZEXT_MEMBASE(REG_SP, src->regoff * 8, d);
 871                         else
 872                                 M_CZEXT(src->regoff, d);
 873                         emit_store(jd, iptr, iptr->dst, d);
 874                         break;
 875
 876                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 877
 878                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 879                         if (src->flags & INMEMORY)
 880                                 M_SSEXT_MEMBASE(REG_SP, src->regoff * 8, d);
 881                         else
 882                                 M_SSEXT(src->regoff, d);
 883                         emit_store(jd, iptr, iptr->dst, d);
 884                         break;
 885
 886
 887                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 888
 889 #if RISC_STYLE
 890                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
 891                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
 892                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
 893                         if (s1 == d)
 894                                 M_IADD(s2, d);
 895                         else if (s2 == d)
 896                                 M_IADD(s1, d);
 897                         else {
 898                                 M_INTMOVE(s1, d);
 899                                 M_IADD(s2, d);
 900                         }
 901                         emit_store(jd, iptr, iptr->dst, d);
 902 #else
 903                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 904                         emit_ialu(cd, ALU_ADD, src, iptr);
 905 #endif
 906                         break;
 907
 908                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 909                                       /* val.i = constant                             */
 910
 911                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 912                         emit_ialuconst(cd, ALU_ADD, src, iptr);
 913                         break;
 914
 915                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 916
 917                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 918                         emit_lalu(cd, ALU_ADD, src, iptr);
 919                         break;
 920
 921                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 922                                       /* val.l = constant                             */
 923
 924                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 925                         emit_laluconst(cd, ALU_ADD, src, iptr);
 926                         break;
 927
 928                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 929
 930                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 931                         if (iptr->dst->flags & INMEMORY) {
 932                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 933                                         if (src->prev->regoff == iptr->dst->regoff) {
 934                                                 emit_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 935                                                 emit_alul_reg_membase(cd, ALU_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 936
 937                                         } else {
 938                                                 emit_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 939                                                 emit_alul_membase_reg(cd, ALU_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 940                                                 emit_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 941                                         }
 942
 943                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 944                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 945                                         emit_alul_membase_reg(cd, ALU_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 946                                         emit_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 947
 948                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 949                                         if (src->prev->regoff == iptr->dst->regoff) {
 950                                                 emit_alul_reg_membase(cd, ALU_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 951
 952                                         } else {
 953                                                 emit_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 954                                                 emit_alul_reg_reg(cd, ALU_SUB, src->regoff, REG_ITMP1);
 955                                                 emit_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 956                                         }
 957
 958                                 } else {
 959                                         emit_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 960                                         emit_alul_reg_membase(cd, ALU_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 961                                 }
 962
 963                         } else {
 964                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 965                                         emit_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 966                                         emit_alul_membase_reg(cd, ALU_SUB, REG_SP, src->regoff * 8, d);
 967
 968                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 969                                         M_INTMOVE(src->prev->regoff, d);
 970                                         emit_alul_membase_reg(cd, ALU_SUB, REG_SP, src->regoff * 8, d);
 971
 972                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 973                                         /* workaround for reg alloc */
 974                                         if (src->regoff == iptr->dst->regoff) {
 975                                                 emit_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 976                                                 emit_alul_reg_reg(cd, ALU_SUB, src->regoff, REG_ITMP1);
 977                                                 M_INTMOVE(REG_ITMP1, d);
 978
 979                                         } else {
 980                                                 emit_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 981                                                 emit_alul_reg_reg(cd, ALU_SUB, src->regoff, d);
 982                                         }
 983
 984                                 } else {
 985                                         /* workaround for reg alloc */
 986                                         if (src->regoff == iptr->dst->regoff) {
 987                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
 988                                                 emit_alul_reg_reg(cd, ALU_SUB, src->regoff, REG_ITMP1);
 989                                                 M_INTMOVE(REG_ITMP1, d);
 990
 991                                         } else {
 992                                                 M_INTMOVE(src->prev->regoff, d);
 993                                                 emit_alul_reg_reg(cd, ALU_SUB, src->regoff, d);
 994                                         }
 995                                 }
 996                         }
 997                         break;
 998
 999                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
1000                                       /* val.i = constant                             */
1001
1002                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1003                         emit_ialuconst(cd, ALU_SUB, src, iptr);
1004                         break;
1005
1006                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1007
1008                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1009                         if (iptr->dst->flags & INMEMORY) {
1010                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1011                                         if (src->prev->regoff == iptr->dst->regoff) {
1012                                                 emit_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1013                                                 emit_alu_reg_membase(cd, ALU_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1014
1015                                         } else {
1016                                                 emit_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1017                                                 emit_alu_membase_reg(cd, ALU_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1018                                                 emit_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1019                                         }
1020
1021                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1022                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1023                                         emit_alu_membase_reg(cd, ALU_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1024                                         emit_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1025
1026                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1027                                         if (src->prev->regoff == iptr->dst->regoff) {
1028                                                 emit_alu_reg_membase(cd, ALU_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1029
1030                                         } else {
1031                                                 emit_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1032                                                 emit_alu_reg_reg(cd, ALU_SUB, src->regoff, REG_ITMP1);
1033                                                 emit_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1034                                         }
1035
1036                                 } else {
1037                                         emit_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
1038                                         emit_alu_reg_membase(cd, ALU_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1039                                 }
1040
1041                         } else {
1042                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1043                                         emit_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1044                                         emit_alu_membase_reg(cd, ALU_SUB, REG_SP, src->regoff * 8, d);
1045
1046                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1047                                         M_INTMOVE(src->prev->regoff, d);
1048                                         emit_alu_membase_reg(cd, ALU_SUB, REG_SP, src->regoff * 8, d);
1049
1050                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1051                                         /* workaround for reg alloc */
1052                                         if (src->regoff == iptr->dst->regoff) {
1053                                                 emit_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1054                                                 emit_alu_reg_reg(cd, ALU_SUB, src->regoff, REG_ITMP1);
1055                                                 M_INTMOVE(REG_ITMP1, d);
1056
1057                                         } else {
1058                                                 emit_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1059                                                 emit_alu_reg_reg(cd, ALU_SUB, src->regoff, d);
1060                                         }
1061
1062                                 } else {
1063                                         /* workaround for reg alloc */
1064                                         if (src->regoff == iptr->dst->regoff) {
1065                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1066                                                 emit_alu_reg_reg(cd, ALU_SUB, src->regoff, REG_ITMP1);
1067                                                 M_INTMOVE(REG_ITMP1, d);
1068
1069                                         } else {
1070                                                 M_INTMOVE(src->prev->regoff, d);
1071                                                 emit_alu_reg_reg(cd, ALU_SUB, src->regoff, d);
1072                                         }
1073                                 }
1074                         }
1075                         break;
1076
1077                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1078                                       /* val.l = constant                             */
1079
1080                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1081                         emit_laluconst(cd, ALU_SUB, src, iptr);
1082                         break;
1083
1084                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1085
1086                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1087                         if (iptr->dst->flags & INMEMORY) {
1088                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1089                                         emit_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1090                                         emit_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1091                                         emit_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1092
1093                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1094                                         emit_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1095                                         emit_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1096                                         emit_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1097
1098                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1099                                         emit_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1100                                         emit_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1101                                         emit_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1102
1103                                 } else {
1104                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1105                                         emit_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1106                                         emit_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1107                                 }
1108
1109                         } else {
1110                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1111                                         emit_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1112                                         emit_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1113
1114                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1115                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1116                                         emit_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1117
1118                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1119                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1120                                         emit_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1121
1122                                 } else {
1123                                         if (src->regoff == iptr->dst->regoff) {
1124                                                 emit_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1125
1126                                         } else {
1127                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1128                                                 emit_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1129                                         }
1130                                 }
1131                         }
1132                         break;
1133
1134                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1135                                       /* val.i = constant                             */
1136
1137                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1138                         if (iptr->dst->flags & INMEMORY) {
1139                                 if (src->flags & INMEMORY) {
1140                                         emit_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1141                                         emit_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1142
1143                                 } else {
1144                                         emit_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1145                                         emit_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1146                                 }
1147
1148                         } else {
1149                                 if (src->flags & INMEMORY) {
1150                                         emit_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1151
1152                                 } else {
1153                                         if (iptr->val.i == 2) {
1154                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1155                                                 emit_alul_reg_reg(cd, ALU_ADD, iptr->dst->regoff, iptr->dst->regoff);
1156
1157                                         } else {
1158                                                 emit_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1159                                         }
1160                                 }
1161                         }
1162                         break;
1163
1164                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1165
1166                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1167                         if (iptr->dst->flags & INMEMORY) {
1168                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1169                                         emit_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1170                                         emit_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1171                                         emit_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1172
1173                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1174                                         emit_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1175                                         emit_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1176                                         emit_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1177
1178                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1179                                         emit_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1180                                         emit_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1181                                         emit_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1182
1183                                 } else {
1184                                         emit_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1185                                         emit_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1186                                         emit_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1187                                 }
1188
1189                         } else {
1190                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1191                                         emit_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1192                                         emit_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1193
1194                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1195                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1196                                         emit_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1197
1198                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1199                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1200                                         emit_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1201
1202                                 } else {
1203                                         if (src->regoff == iptr->dst->regoff) {
1204                                                 emit_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1205
1206                                         } else {
1207                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1208                                                 emit_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1209                                         }
1210                                 }
1211                         }
1212                         break;
1213
1214                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1215                                       /* val.l = constant                             */
1216
1217                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1218                         if (iptr->dst->flags & INMEMORY) {
1219                                 if (src->flags & INMEMORY) {
1220                                         if (IS_IMM32(iptr->val.l)) {
1221                                                 emit_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1222
1223                                         } else {
1224                                                 emit_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1225                                                 emit_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1226                                         }
1227                                         emit_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1228
1229                                 } else {
1230                                         if (IS_IMM32(iptr->val.l)) {
1231                                                 emit_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1232
1233                                         } else {
1234                                                 emit_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1235                                                 emit_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1236                                         }
1237                                         emit_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1238                                 }
1239
1240                         } else {
1241                                 if (src->flags & INMEMORY) {
1242                                         if (IS_IMM32(iptr->val.l)) {
1243                                                 emit_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1244
1245                                         } else {
1246                                                 emit_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1247                                                 emit_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1248                                         }
1249
1250                                 } else {
1251                                         /* should match in many cases */
1252                                         if (iptr->val.l == 2) {
1253                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1254                                                 emit_alul_reg_reg(cd, ALU_ADD, iptr->dst->regoff, iptr->dst->regoff);
1255
1256                                         } else {
1257                                                 if (IS_IMM32(iptr->val.l)) {
1258                                                         emit_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1259
1260                                                 } else {
1261                                                         emit_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1262                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1263                                                         emit_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1264                                                 }
1265                                         }
1266                                 }
1267                         }
1268                         break;
1269
1270                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1271
1272                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1273                 if (src->prev->flags & INMEMORY) {
1274                                 emit_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1275
1276                         } else {
1277                                 M_INTMOVE(src->prev->regoff, RAX);
1278                         }
1279
1280                         if (src->flags & INMEMORY) {
1281                                 emit_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1282
1283                         } else {
1284                                 M_INTMOVE(src->regoff, REG_ITMP3);
1285                         }
1286                         gen_div_check(src);
1287
1288                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1289                         emit_jcc(cd, CC_NE, 4 + 6);
1290                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
1291                         emit_jcc(cd, CC_E, 3 + 1 + 3);                /* 6 bytes */
1292
1293                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1294                         emit_cltd(cd);
1295                         emit_idivl_reg(cd, REG_ITMP3);
1296
1297                         if (iptr->dst->flags & INMEMORY) {
1298                                 emit_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1299                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1300
1301                         } else {
1302                                 M_INTMOVE(RAX, iptr->dst->regoff);
1303
1304                                 if (iptr->dst->regoff != RDX) {
1305                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1306                                 }
1307                         }
1308                         break;
1309
1310                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1311                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1312                         if (src->prev->flags & INMEMORY) {
1313                                 emit_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1314
1315                         } else {
1316                                 M_INTMOVE(src->prev->regoff, RAX);
1317                         }
1318
1319                         if (src->flags & INMEMORY) {
1320                                 emit_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1321
1322                         } else {
1323                                 M_INTMOVE(src->regoff, REG_ITMP3);
1324                         }
1325                         gen_div_check(src);
1326
1327                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1328
1329                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1330                         emit_jcc(cd, CC_NE, 2 + 4 + 6);
1331
1332
1333                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
1334                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
1335                         emit_jcc(cd, CC_E, 1 + 3);                    /* 6 bytes */
1336
1337                         emit_cltd(cd);
1338                         emit_idivl_reg(cd, REG_ITMP3);
1339
1340                         if (iptr->dst->flags & INMEMORY) {
1341                                 emit_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1342                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1343
1344                         } else {
1345                                 M_INTMOVE(RDX, iptr->dst->regoff);
1346
1347                                 if (iptr->dst->regoff != RDX) {
1348                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1349                                 }
1350                         }
1351                         break;
1352
1353                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1354                                       /* val.i = constant                             */
1355
1356                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1357                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1358                         M_INTMOVE(s1, REG_ITMP1);
1359                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1360                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1361                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
1362                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->val.i, REG_ITMP1);
1363                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1364                         emit_store(jd, iptr, iptr->dst, d);
1365                         break;
1366
1367                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1368                                       /* val.i = constant                             */
1369
1370                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1371                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1372                         M_INTMOVE(s1, REG_ITMP1);
1373                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1374                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1375                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
1376                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->val.i), REG_ITMP2);
1377                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
1378                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1379                         emit_store(jd, iptr, iptr->dst, d);
1380                         break;
1381
1382
1383                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1384
1385                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1386
1387                 if (src->prev->flags & INMEMORY) {
1388                                 M_LLD(RAX, REG_SP, src->prev->regoff * 8);
1389
1390                         } else {
1391                                 M_INTMOVE(src->prev->regoff, RAX);
1392                         }
1393
1394                         if (src->flags & INMEMORY) {
1395                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1396
1397                         } else {
1398                                 M_INTMOVE(src->regoff, REG_ITMP3);
1399                         }
1400                         gen_div_check(src);
1401
1402                         /* check as described in jvm spec */
1403                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1404                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, RAX);
1405                         M_BNE(4 + 6);
1406                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1407                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
1408
1409                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1410                         emit_cqto(cd);
1411                         emit_idiv_reg(cd, REG_ITMP3);
1412
1413                         if (iptr->dst->flags & INMEMORY) {
1414                                 M_LST(RAX, REG_SP, iptr->dst->regoff * 8);
1415                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1416
1417                         } else {
1418                                 M_INTMOVE(RAX, iptr->dst->regoff);
1419
1420                                 if (iptr->dst->regoff != RDX) {
1421                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1422                                 }
1423                         }
1424                         break;
1425
1426                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1427
1428                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1429                         if (src->prev->flags & INMEMORY) {
1430                                 M_LLD(REG_ITMP1, REG_SP, src->prev->regoff * 8);
1431
1432                         } else {
1433                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1434                         }
1435
1436                         if (src->flags & INMEMORY) {
1437                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1438
1439                         } else {
1440                                 M_INTMOVE(src->regoff, REG_ITMP3);
1441                         }
1442                         gen_div_check(src);
1443
1444                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1445
1446                         /* check as described in jvm spec */
1447                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1448                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP1);
1449                         M_BNE(3 + 4 + 6);
1450
1451 #if 0
1452                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
1453 #endif
1454                         M_XOR(RDX, RDX);                                       /* 3 bytes */
1455                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1456                         M_BEQ(2 + 3);                                          /* 6 bytes */
1457
1458                         emit_cqto(cd);
1459                         emit_idiv_reg(cd, REG_ITMP3);
1460
1461                         if (iptr->dst->flags & INMEMORY) {
1462                                 M_LST(RDX, REG_SP, iptr->dst->regoff * 8);
1463                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1464
1465                         } else {
1466                                 M_INTMOVE(RDX, iptr->dst->regoff);
1467
1468                                 if (iptr->dst->regoff != RDX) {
1469                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1470                                 }
1471                         }
1472                         break;
1473
1474                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1475                                       /* val.i = constant                             */
1476
1477                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1478                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1479                         M_INTMOVE(s1, REG_ITMP1);
1480                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1481                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1482                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
1483                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->val.i, REG_ITMP1);
1484                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1485                         emit_store(jd, iptr, iptr->dst, d);
1486                         break;
1487
1488                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1489                                       /* val.l = constant                             */
1490
1491                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1492                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1493                         M_INTMOVE(s1, REG_ITMP1);
1494                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1495                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1496                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
1497                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->val.i), REG_ITMP2);
1498                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
1499                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1500                         emit_store(jd, iptr, iptr->dst, d);
1501                         break;
1502
1503                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1504
1505                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1506                         emit_ishift(cd, SHIFT_SHL, src, iptr);
1507                         break;
1508
1509                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1510                                       /* val.i = constant                             */
1511
1512                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1513                         emit_ishiftconst(cd, SHIFT_SHL, src, iptr);
1514                         break;
1515
1516                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1517
1518                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1519                         emit_ishift(cd, SHIFT_SAR, src, iptr);
1520                         break;
1521
1522                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1523                                       /* val.i = constant                             */
1524
1525                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1526                         emit_ishiftconst(cd, SHIFT_SAR, src, iptr);
1527                         break;
1528
1529                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1530
1531                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1532                         emit_ishift(cd, SHIFT_SHR, src, iptr);
1533                         break;
1534
1535                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1536                                       /* val.i = constant                             */
1537
1538                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1539                         emit_ishiftconst(cd, SHIFT_SHR, src, iptr);
1540                         break;
1541
1542                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1543
1544                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1545                         emit_lshift(cd, SHIFT_SHL, src, iptr);
1546                         break;
1547
1548         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1549                                           /* val.i = constant                             */
1550
1551                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1552                         emit_lshiftconst(cd, SHIFT_SHL, src, iptr);
1553                         break;
1554
1555                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1556
1557                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1558                         emit_lshift(cd, SHIFT_SAR, src, iptr);
1559                         break;
1560
1561                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1562                                       /* val.i = constant                             */
1563
1564                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1565                         emit_lshiftconst(cd, SHIFT_SAR, src, iptr);
1566                         break;
1567
1568                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1569
1570                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1571                         emit_lshift(cd, SHIFT_SHR, src, iptr);
1572                         break;
1573
1574                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1575                                       /* val.l = constant                             */
1576
1577                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1578                         emit_lshiftconst(cd, SHIFT_SHR, src, iptr);
1579                         break;
1580
1581                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1582
1583                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1584                         emit_ialu(cd, ALU_AND, src, iptr);
1585                         break;
1586
1587                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1588                                       /* val.i = constant                             */
1589
1590                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1591                         emit_ialuconst(cd, ALU_AND, src, iptr);
1592                         break;
1593
1594                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1595
1596                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1597                         emit_lalu(cd, ALU_AND, src, iptr);
1598                         break;
1599
1600                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1601                                       /* val.l = constant                             */
1602
1603                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1604                         emit_laluconst(cd, ALU_AND, src, iptr);
1605                         break;
1606
1607                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1608
1609                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1610                         emit_ialu(cd, ALU_OR, src, iptr);
1611                         break;
1612
1613                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1614                                       /* val.i = constant                             */
1615
1616                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1617                         emit_ialuconst(cd, ALU_OR, src, iptr);
1618                         break;
1619
1620                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1621
1622                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1623                         emit_lalu(cd, ALU_OR, src, iptr);
1624                         break;
1625
1626                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1627                                       /* val.l = constant                             */
1628
1629                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1630                         emit_laluconst(cd, ALU_OR, src, iptr);
1631                         break;
1632
1633                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1634
1635                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1636                         emit_ialu(cd, ALU_XOR, src, iptr);
1637                         break;
1638
1639                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1640                                       /* val.i = constant                             */
1641
1642                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1643                         emit_ialuconst(cd, ALU_XOR, src, iptr);
1644                         break;
1645
1646                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1647
1648                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1649                         emit_lalu(cd, ALU_XOR, src, iptr);
1650                         break;
1651
1652                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1653                                       /* val.l = constant                             */
1654
1655                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1656                         emit_laluconst(cd, ALU_XOR, src, iptr);
1657                         break;
1658
1659
1660                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1661                                       /* op1 = variable, val.i = constant             */
1662
1663                         /* using inc and dec is definitely faster than add -- tested      */
1664                         /* with sieve                                                     */
1665
1666                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1667                         d = var->regoff;
1668                         if (var->flags & INMEMORY) {
1669                                 if (iptr->val.i == 1) {
1670                                         emit_incl_membase(cd, REG_SP, d * 8);
1671
1672                                 } else if (iptr->val.i == -1) {
1673                                         emit_decl_membase(cd, REG_SP, d * 8);
1674
1675                                 } else {
1676                                         emit_alul_imm_membase(cd, ALU_ADD, iptr->val.i, REG_SP, d * 8);
1677                                 }
1678
1679                         } else {
1680                                 if (iptr->val.i == 1) {
1681                                         emit_incl_reg(cd, d);
1682
1683                                 } else if (iptr->val.i == -1) {
1684                                         emit_decl_reg(cd, d);
1685
1686                                 } else {
1687                                         emit_alul_imm_reg(cd, ALU_ADD, iptr->val.i, d);
1688                                 }
1689                         }
1690                         break;
1691
1692
1693                 /* floating operations ************************************************/
1694
1695                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1696
1697                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1698                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1699                         disp = dseg_adds4(cd, 0x80000000);
1700                         M_FLTMOVE(s1, d);
1701                         emit_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1702                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1703                         emit_store(jd, iptr, iptr->dst, d);
1704                         break;
1705
1706                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1707
1708                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1709                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1710                         disp = dseg_adds8(cd, 0x8000000000000000);
1711                         M_FLTMOVE(s1, d);
1712                         emit_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1713                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1714                         emit_store(jd, iptr, iptr->dst, d);
1715                         break;
1716
1717                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1718
1719                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1720                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1721                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1722                         if (s1 == d) {
1723                                 emit_addss_reg_reg(cd, s2, d);
1724                         } else if (s2 == d) {
1725                                 emit_addss_reg_reg(cd, s1, d);
1726                         } else {
1727                                 M_FLTMOVE(s1, d);
1728                                 emit_addss_reg_reg(cd, s2, d);
1729                         }
1730                         emit_store(jd, iptr, iptr->dst, d);
1731                         break;
1732
1733                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1734
1735                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1736                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1737                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1738                         if (s1 == d) {
1739                                 emit_addsd_reg_reg(cd, s2, d);
1740                         } else if (s2 == d) {
1741                                 emit_addsd_reg_reg(cd, s1, d);
1742                         } else {
1743                                 M_FLTMOVE(s1, d);
1744                                 emit_addsd_reg_reg(cd, s2, d);
1745                         }
1746                         emit_store(jd, iptr, iptr->dst, d);
1747                         break;
1748
1749                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1750
1751                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1752                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1753                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1754                         if (s2 == d) {
1755                                 M_FLTMOVE(s2, REG_FTMP2);
1756                                 s2 = REG_FTMP2;
1757                         }
1758                         M_FLTMOVE(s1, d);
1759                         emit_subss_reg_reg(cd, s2, d);
1760                         emit_store(jd, iptr, iptr->dst, d);
1761                         break;
1762
1763                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1764
1765                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1766                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1767                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1768                         if (s2 == d) {
1769                                 M_FLTMOVE(s2, REG_FTMP2);
1770                                 s2 = REG_FTMP2;
1771                         }
1772                         M_FLTMOVE(s1, d);
1773                         emit_subsd_reg_reg(cd, s2, d);
1774                         emit_store(jd, iptr, iptr->dst, d);
1775                         break;
1776
1777                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1778
1779                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1780                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1781                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1782                         if (s1 == d) {
1783                                 emit_mulss_reg_reg(cd, s2, d);
1784                         } else if (s2 == d) {
1785                                 emit_mulss_reg_reg(cd, s1, d);
1786                         } else {
1787                                 M_FLTMOVE(s1, d);
1788                                 emit_mulss_reg_reg(cd, s2, d);
1789                         }
1790                         emit_store(jd, iptr, iptr->dst, d);
1791                         break;
1792
1793                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1794
1795                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1796                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1797                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1798                         if (s1 == d) {
1799                                 emit_mulsd_reg_reg(cd, s2, d);
1800                         } else if (s2 == d) {
1801                                 emit_mulsd_reg_reg(cd, s1, d);
1802                         } else {
1803                                 M_FLTMOVE(s1, d);
1804                                 emit_mulsd_reg_reg(cd, s2, d);
1805                         }
1806                         emit_store(jd, iptr, iptr->dst, d);
1807                         break;
1808
1809                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1810
1811                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1812                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1813                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1814                         if (s2 == d) {
1815                                 M_FLTMOVE(s2, REG_FTMP2);
1816                                 s2 = REG_FTMP2;
1817                         }
1818                         M_FLTMOVE(s1, d);
1819                         emit_divss_reg_reg(cd, s2, d);
1820                         emit_store(jd, iptr, iptr->dst, d);
1821                         break;
1822
1823                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1824
1825                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1826                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1827                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1828                         if (s2 == d) {
1829                                 M_FLTMOVE(s2, REG_FTMP2);
1830                                 s2 = REG_FTMP2;
1831                         }
1832                         M_FLTMOVE(s1, d);
1833                         emit_divsd_reg_reg(cd, s2, d);
1834                         emit_store(jd, iptr, iptr->dst, d);
1835                         break;
1836
1837                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1838
1839                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1840                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1841                         emit_cvtsi2ss_reg_reg(cd, s1, d);
1842                         emit_store(jd, iptr, iptr->dst, d);
1843                         break;
1844
1845                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1846
1847                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1848                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1849                         emit_cvtsi2sd_reg_reg(cd, s1, d);
1850                         emit_store(jd, iptr, iptr->dst, d);
1851                         break;
1852
1853                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1854
1855                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1856                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1857                         emit_cvtsi2ssq_reg_reg(cd, s1, d);
1858                         emit_store(jd, iptr, iptr->dst, d);
1859                         break;
1860
1861                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1862
1863                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1864                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1865                         emit_cvtsi2sdq_reg_reg(cd, s1, d);
1866                         emit_store(jd, iptr, iptr->dst, d);
1867                         break;
1868
1869                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1870
1871                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1872                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1873                         emit_cvttss2si_reg_reg(cd, s1, d);
1874                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, d);    /* corner cases */
1875                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1876                         emit_jcc(cd, CC_NE, a);
1877                         M_FLTMOVE(s1, REG_FTMP1);
1878                         emit_mov_imm_reg(cd, (ptrint) asm_builtin_f2i, REG_ITMP2);
1879                         emit_call_reg(cd, REG_ITMP2);
1880                         M_INTMOVE(REG_RESULT, d);
1881                         emit_store(jd, iptr, iptr->dst, d);
1882                         break;
1883
1884                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1885
1886                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1887                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1888                         emit_cvttsd2si_reg_reg(cd, s1, d);
1889                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, d);    /* corner cases */
1890                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1891                         emit_jcc(cd, CC_NE, a);
1892                         M_FLTMOVE(s1, REG_FTMP1);
1893                         emit_mov_imm_reg(cd, (ptrint) asm_builtin_d2i, REG_ITMP2);
1894                         emit_call_reg(cd, REG_ITMP2);
1895                         M_INTMOVE(REG_RESULT, d);
1896                         emit_store(jd, iptr, iptr->dst, d);
1897                         break;
1898
1899                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1900
1901                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1902                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1903                         emit_cvttss2siq_reg_reg(cd, s1, d);
1904                         emit_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1905                         emit_alu_reg_reg(cd, ALU_CMP, REG_ITMP2, d);     /* corner cases */
1906                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1907                         emit_jcc(cd, CC_NE, a);
1908                         M_FLTMOVE(s1, REG_FTMP1);
1909                         emit_mov_imm_reg(cd, (ptrint) asm_builtin_f2l, REG_ITMP2);
1910                         emit_call_reg(cd, REG_ITMP2);
1911                         M_INTMOVE(REG_RESULT, d);
1912                         emit_store(jd, iptr, iptr->dst, d);
1913                         break;
1914
1915                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1916
1917                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1918                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1919                         emit_cvttsd2siq_reg_reg(cd, s1, d);
1920                         emit_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1921                         emit_alu_reg_reg(cd, ALU_CMP, REG_ITMP2, d);     /* corner cases */
1922                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1923                         emit_jcc(cd, CC_NE, a);
1924                         M_FLTMOVE(s1, REG_FTMP1);
1925                         emit_mov_imm_reg(cd, (ptrint) asm_builtin_d2l, REG_ITMP2);
1926                         emit_call_reg(cd, REG_ITMP2);
1927                         M_INTMOVE(REG_RESULT, d);
1928                         emit_store(jd, iptr, iptr->dst, d);
1929                         break;
1930
1931                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1932
1933                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1934                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1935                         emit_cvtss2sd_reg_reg(cd, s1, d);
1936                         emit_store(jd, iptr, iptr->dst, d);
1937                         break;
1938
1939                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1940
1941                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1942                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1943                         emit_cvtsd2ss_reg_reg(cd, s1, d);
1944                         emit_store(jd, iptr, iptr->dst, d);
1945                         break;
1946
1947                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1948                                           /* == => 0, < => 1, > => -1 */
1949
1950                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1951                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1952                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1953                         M_CLR(d);
1954                         M_MOV_IMM(1, REG_ITMP1);
1955                         M_MOV_IMM(-1, REG_ITMP2);
1956                         emit_ucomiss_reg_reg(cd, s1, s2);
1957                         M_CMOVB(REG_ITMP1, d);
1958                         M_CMOVA(REG_ITMP2, d);
1959                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1960                         emit_store(jd, iptr, iptr->dst, d);
1961                         break;
1962
1963                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1964                                           /* == => 0, < => 1, > => -1 */
1965
1966                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1967                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1968                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1969                         M_CLR(d);
1970                         M_MOV_IMM(1, REG_ITMP1);
1971                         M_MOV_IMM(-1, REG_ITMP2);
1972                         emit_ucomiss_reg_reg(cd, s1, s2);
1973                         M_CMOVB(REG_ITMP1, d);
1974                         M_CMOVA(REG_ITMP2, d);
1975                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1976                         emit_store(jd, iptr, iptr->dst, d);
1977                         break;
1978
1979                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1980                                           /* == => 0, < => 1, > => -1 */
1981
1982                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1983                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1984                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1985                         M_CLR(d);
1986                         M_MOV_IMM(1, REG_ITMP1);
1987                         M_MOV_IMM(-1, REG_ITMP2);
1988                         emit_ucomisd_reg_reg(cd, s1, s2);
1989                         M_CMOVB(REG_ITMP1, d);
1990                         M_CMOVA(REG_ITMP2, d);
1991                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1992                         emit_store(jd, iptr, iptr->dst, d);
1993                         break;
1994
1995                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1996                                           /* == => 0, < => 1, > => -1 */
1997
1998                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1999                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
2000                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2001                         M_CLR(d);
2002                         M_MOV_IMM(1, REG_ITMP1);
2003                         M_MOV_IMM(-1, REG_ITMP2);
2004                         emit_ucomisd_reg_reg(cd, s1, s2);
2005                         M_CMOVB(REG_ITMP1, d);
2006                         M_CMOVA(REG_ITMP2, d);
2007                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
2008                         emit_store(jd, iptr, iptr->dst, d);
2009                         break;
2010
2011
2012                 /* memory operations **************************************************/
2013
2014                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
2015
2016                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2017                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2018                         gen_nullptr_check(s1);
2019                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
2020                         emit_store(jd, iptr, iptr->dst, d);
2021                         break;
2022
2023                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2024
2025                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2026                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2027                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2028                         if (iptr->op1 == 0) {
2029                                 gen_nullptr_check(s1);
2030                                 gen_bound_check;
2031                         }
2032                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2033                         emit_store(jd, iptr, iptr->dst, d);
2034                         break;
2035
2036                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2037
2038                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2039                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2040                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2041                         if (iptr->op1 == 0) {
2042                                 gen_nullptr_check(s1);
2043                                 gen_bound_check;
2044                         }
2045                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2046                         emit_store(jd, iptr, iptr->dst, d);
2047                         break;
2048
2049                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2050
2051                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2052                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2053                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2054                         if (iptr->op1 == 0) {
2055                                 gen_nullptr_check(s1);
2056                                 gen_bound_check;
2057                         }
2058                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2059                         emit_store(jd, iptr, iptr->dst, d);
2060                         break;
2061
2062                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
2063
2064                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2065                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2066                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2067                         if (iptr->op1 == 0) {
2068                                 gen_nullptr_check(s1);
2069                                 gen_bound_check;
2070                         }
2071                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
2072                         emit_store(jd, iptr, iptr->dst, d);
2073                         break;
2074
2075                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
2076
2077                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2078                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2079                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2080                         if (iptr->op1 == 0) {
2081                                 gen_nullptr_check(s1);
2082                                 gen_bound_check;
2083                         }
2084                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
2085                         emit_store(jd, iptr, iptr->dst, d);
2086                         break;
2087
2088                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
2089
2090                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2091                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2092                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
2093                         if (iptr->op1 == 0) {
2094                                 gen_nullptr_check(s1);
2095                                 gen_bound_check;
2096                         }
2097                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2098                         emit_store(jd, iptr, iptr->dst, d);
2099                         break;
2100
2101                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2102
2103                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2104                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2105                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
2106                         if (iptr->op1 == 0) {
2107                                 gen_nullptr_check(s1);
2108                                 gen_bound_check;
2109                         }
2110                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2111                         emit_store(jd, iptr, iptr->dst, d);
2112                         break;
2113
2114                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
2115
2116                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2117                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2118                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2119                         if (iptr->op1 == 0) {
2120                                 gen_nullptr_check(s1);
2121                                 gen_bound_check;
2122                         }
2123                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
2124                         emit_store(jd, iptr, iptr->dst, d);
2125                         break;
2126
2127
2128                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2129
2130                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2131                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2132                         if (iptr->op1 == 0) {
2133                                 gen_nullptr_check(s1);
2134                                 gen_bound_check;
2135                         }
2136                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2137                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2138                         break;
2139
2140                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2141
2142                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2143                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2144                         if (iptr->op1 == 0) {
2145                                 gen_nullptr_check(s1);
2146                                 gen_bound_check;
2147                         }
2148                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2149                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2150                         break;
2151
2152                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2153
2154                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2155                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2156                         if (iptr->op1 == 0) {
2157                                 gen_nullptr_check(s1);
2158                                 gen_bound_check;
2159                         }
2160                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2161                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2162                         break;
2163
2164                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2165
2166                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2167                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2168                         if (iptr->op1 == 0) {
2169                                 gen_nullptr_check(s1);
2170                                 gen_bound_check;
2171                         }
2172                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2173                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2174                         break;
2175
2176                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2177
2178                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2179                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2180                         if (iptr->op1 == 0) {
2181                                 gen_nullptr_check(s1);
2182                                 gen_bound_check;
2183                         }
2184                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2185                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2186                         break;
2187
2188                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2189
2190                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2191                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2192                         if (iptr->op1 == 0) {
2193                                 gen_nullptr_check(s1);
2194                                 gen_bound_check;
2195                         }
2196                         s3 = emit_load_s3(jd, iptr, src, REG_FTMP3);
2197                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2198                         break;
2199
2200                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2201
2202                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2203                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2204                         if (iptr->op1 == 0) {
2205                                 gen_nullptr_check(s1);
2206                                 gen_bound_check;
2207                         }
2208                         s3 = emit_load_s3(jd, iptr, src, REG_FTMP3);
2209                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2210                         break;
2211
2212                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2213
2214                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2215                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2216                         if (iptr->op1 == 0) {
2217                                 gen_nullptr_check(s1);
2218                                 gen_bound_check;
2219                         }
2220                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2221
2222                         M_MOV(s1, rd->argintregs[0]);
2223                         M_MOV(s3, rd->argintregs[1]);
2224                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
2225                         M_CALL(REG_ITMP1);
2226                         M_TEST(REG_RESULT);
2227                         M_BEQ(0);
2228                         codegen_add_arraystoreexception_ref(cd);
2229
2230                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2231                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2232                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2233                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2234                         break;
2235
2236
2237                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2238
2239                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2240                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2241                         if (iptr->op1 == 0) {
2242                                 gen_nullptr_check(s1);
2243                                 gen_bound_check;
2244                         }
2245                         emit_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2246                         break;
2247
2248                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2249
2250                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2251                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2252                         if (iptr->op1 == 0) {
2253                                 gen_nullptr_check(s1);
2254                                 gen_bound_check;
2255                         }
2256                         emit_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2257                         break;
2258
2259                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2260
2261                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2262                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2263                         if (iptr->op1 == 0) {
2264                                 gen_nullptr_check(s1);
2265                                 gen_bound_check;
2266                         }
2267                         emit_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2268                         break;
2269
2270                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2271
2272                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2273                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2274                         if (iptr->op1 == 0) {
2275                                 gen_nullptr_check(s1);
2276                                 gen_bound_check;
2277                         }
2278                         emit_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2279                         break;
2280
2281                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2282
2283                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2284                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2285                         if (iptr->op1 == 0) {
2286                                 gen_nullptr_check(s1);
2287                                 gen_bound_check;
2288                         }
2289
2290                         if (IS_IMM32(iptr->val.l)) {
2291                                 emit_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2292                         } else {
2293                                 emit_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2294                                 emit_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2295                         }
2296                         break;
2297
2298                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2299
2300                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2301                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2302                         if (iptr->op1 == 0) {
2303                                 gen_nullptr_check(s1);
2304                                 gen_bound_check;
2305                         }
2306                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2307                         break;
2308
2309
2310                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2311                                       /* op1 = type, val.a = field address            */
2312
2313                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2314                                 disp = dseg_addaddress(cd, NULL);
2315
2316 /*                              PROFILE_CYCLE_STOP; */
2317
2318                                 codegen_addpatchref(cd, PATCHER_get_putstatic,
2319                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), disp);
2320
2321                                 if (opt_showdisassemble) {
2322                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2323                                 }
2324
2325 /*                              PROFILE_CYCLE_START; */
2326
2327                         } else {
2328                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr);
2329
2330                                 disp = dseg_addaddress(cd, &(fi->value));
2331
2332                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2333                                         PROFILE_CYCLE_STOP;
2334
2335                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2336
2337                                         if (opt_showdisassemble) {
2338                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2339                                         }
2340
2341                                         PROFILE_CYCLE_START;
2342                                 }
2343                         }
2344
2345                         /* This approach is much faster than moving the field
2346                            address inline into a register. */
2347
2348                         M_ALD(REG_ITMP2, RIP, -(((ptrint) cd->mcodeptr + 7) -
2349                                                                         (ptrint) cd->mcodebase) + disp);
2350
2351                         switch (iptr->op1) {
2352                         case TYPE_INT:
2353                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2354                                 M_ILD(d, REG_ITMP2, 0);
2355                                 emit_store(jd, iptr, iptr->dst, d);
2356                                 break;
2357                         case TYPE_LNG:
2358                         case TYPE_ADR:
2359                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2360                                 M_LLD(d, REG_ITMP2, 0);
2361                                 emit_store(jd, iptr, iptr->dst, d);
2362                                 break;
2363                         case TYPE_FLT:
2364                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2365                                 emit_movss_membase_reg(cd, REG_ITMP2, 0, d);
2366                                 emit_store(jd, iptr, iptr->dst, d);
2367                                 break;
2368                         case TYPE_DBL:
2369                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2370                                 emit_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2371                                 emit_store(jd, iptr, iptr->dst, d);
2372                                 break;
2373                         }
2374                         break;
2375
2376                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2377                                       /* op1 = type, val.a = field address            */
2378
2379                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2380                                 disp = dseg_addaddress(cd, NULL);
2381
2382 /*                              PROFILE_CYCLE_STOP; */
2383
2384                                 codegen_addpatchref(cd, PATCHER_get_putstatic,
2385                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), disp);
2386
2387                                 if (opt_showdisassemble) {
2388                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2389                                 }
2390
2391 /*                              PROFILE_CYCLE_START; */
2392
2393                         } else {
2394                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr);
2395
2396                                 disp = dseg_addaddress(cd, &(fi->value));
2397
2398                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2399                                         PROFILE_CYCLE_STOP;
2400
2401                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2402
2403                                         if (opt_showdisassemble) {
2404                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2405                                         }
2406
2407                                         PROFILE_CYCLE_START;
2408                                 }
2409                         }
2410
2411                         /* This approach is much faster than moving the field
2412                            address inline into a register. */
2413
2414                         M_ALD(REG_ITMP2, RIP, -(((ptrint) cd->mcodeptr + 7) -
2415                                                                         (ptrint) cd->mcodebase) + disp);
2416
2417                         switch (iptr->op1) {
2418                         case TYPE_INT:
2419                                 s2 = emit_load_s2(jd, iptr, src, REG_ITMP1);
2420                                 M_IST(s2, REG_ITMP2, 0);
2421                                 break;
2422                         case TYPE_LNG:
2423                         case TYPE_ADR:
2424                                 s2 = emit_load_s2(jd, iptr, src, REG_ITMP1);
2425                                 M_LST(s2, REG_ITMP2, 0);
2426                                 break;
2427                         case TYPE_FLT:
2428                                 s2 = emit_load_s2(jd, iptr, src, REG_FTMP1);
2429                                 emit_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2430                                 break;
2431                         case TYPE_DBL:
2432                                 s2 = emit_load_s2(jd, iptr, src, REG_FTMP1);
2433                                 emit_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2434                                 break;
2435                         }
2436                         break;
2437
2438                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2439                                           /* val = value (in current instruction)     */
2440                                           /* op1 = type, val.a = field address (in    */
2441                                           /* following NOP)                           */
2442
2443                         if (INSTRUCTION_IS_UNRESOLVED(iptr + 1)) {
2444                                 disp = dseg_addaddress(cd, NULL);
2445
2446 /*                              PROFILE_CYCLE_STOP; */
2447
2448                                 codegen_addpatchref(cd, PATCHER_get_putstatic,
2449                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr + 1), disp);
2450
2451                                 if (opt_showdisassemble) {
2452                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2453                                 }
2454
2455 /*                              PROFILE_CYCLE_START; */
2456
2457                         } else {
2458                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr + 1);
2459
2460                                 disp = dseg_addaddress(cd, &(fi->value));
2461
2462                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2463                                         PROFILE_CYCLE_STOP;
2464
2465                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2466
2467                                         if (opt_showdisassemble) {
2468                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2469                                         }
2470
2471                                         PROFILE_CYCLE_START;
2472                                 }
2473                         }
2474
2475                         /* This approach is much faster than moving the field
2476                            address inline into a register. */
2477
2478                         M_ALD(REG_ITMP1, RIP, -(((ptrint) cd->mcodeptr + 7) -
2479                                                                         (ptrint) cd->mcodebase) + disp);
2480
2481                         switch (iptr->op1) {
2482                         case TYPE_INT:
2483                         case TYPE_FLT:
2484                                 M_IST_IMM(iptr->val.i, REG_ITMP1, 0);
2485                                 break;
2486                         case TYPE_LNG:
2487                         case TYPE_ADR:
2488                         case TYPE_DBL:
2489                                 if (IS_IMM32(iptr->val.l)) {
2490                                         M_LST_IMM32(iptr->val.l, REG_ITMP1, 0);
2491                                 } else {
2492                                         M_IST_IMM(iptr->val.l, REG_ITMP1, 0);
2493                                         M_IST_IMM(iptr->val.l >> 32, REG_ITMP1, 4);
2494                                 }
2495                                 break;
2496                         }
2497                         break;
2498
2499                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2500                                       /* op1 = type, val.i = field offset             */
2501
2502                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2503                         gen_nullptr_check(s1);
2504
2505                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2506 /*                              PROFILE_CYCLE_STOP; */
2507
2508                                 codegen_addpatchref(cd, PATCHER_get_putfield,
2509                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), 0);
2510
2511                                 if (opt_showdisassemble) {
2512                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2513                                 }
2514
2515 /*                              PROFILE_CYCLE_START; */
2516
2517                                 disp = 0;
2518
2519                         } else {
2520                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr)->offset;
2521                         }
2522
2523                         switch (iptr->op1) {
2524                         case TYPE_INT:
2525                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2526                                 M_ILD32(d, s1, disp);
2527                                 emit_store(jd, iptr, iptr->dst, d);
2528                                 break;
2529                         case TYPE_LNG:
2530                         case TYPE_ADR:
2531                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2532                                 M_LLD32(d, s1, disp);
2533                                 emit_store(jd, iptr, iptr->dst, d);
2534                                 break;
2535                         case TYPE_FLT:
2536                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2537                                 emit_movss_membase32_reg(cd, s1, disp, d);
2538                                 emit_store(jd, iptr, iptr->dst, d);
2539                                 break;
2540                         case TYPE_DBL:
2541                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2542                                 emit_movsd_membase32_reg(cd, s1, disp, d);
2543                                 emit_store(jd, iptr, iptr->dst, d);
2544                                 break;
2545                         }
2546                         break;
2547
2548                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2549                                       /* op1 = type, val.i = field offset             */
2550
2551                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2552                         gen_nullptr_check(s1);
2553
2554                         if (IS_INT_LNG_TYPE(iptr->op1)) {
2555                                 s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2556                         } else {
2557                                 s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
2558                         }
2559
2560                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2561 /*                              PROFILE_CYCLE_STOP; */
2562
2563                                 codegen_addpatchref(cd, PATCHER_get_putfield,
2564                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), 0);
2565
2566                                 if (opt_showdisassemble) {
2567                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2568                                 }
2569
2570 /*                              PROFILE_CYCLE_START; */
2571
2572                                 disp = 0;
2573
2574                         } else {
2575                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr)->offset;
2576                         }
2577
2578                         switch (iptr->op1) {
2579                         case TYPE_INT:
2580                                 M_IST32(s2, s1, disp);
2581                                 break;
2582                         case TYPE_LNG:
2583                         case TYPE_ADR:
2584                                 M_LST32(s2, s1, disp);
2585                                 break;
2586                         case TYPE_FLT:
2587                                 emit_movss_reg_membase32(cd, s2, s1, disp);
2588                                 break;
2589                         case TYPE_DBL:
2590                                 emit_movsd_reg_membase32(cd, s2, s1, disp);
2591                                 break;
2592                         }
2593                         break;
2594
2595                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2596                                           /* val = value (in current instruction)     */
2597                                           /* op1 = type, val.a = field address (in    */
2598                                           /* following NOP)                           */
2599
2600                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2601                         gen_nullptr_check(s1);
2602
2603                         if (INSTRUCTION_IS_UNRESOLVED(iptr + 1)) {
2604 /*                              PROFILE_CYCLE_STOP; */
2605
2606                                 codegen_addpatchref(cd, PATCHER_putfieldconst,
2607                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr + 1), 0);
2608
2609                                 if (opt_showdisassemble) {
2610                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2611                                 }
2612
2613 /*                              PROFILE_CYCLE_START; */
2614
2615                                 disp = 0;
2616
2617                         } else {
2618                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr + 1)->offset;
2619                         }
2620
2621                         switch (iptr->op1) {
2622                         case TYPE_INT:
2623                         case TYPE_FLT:
2624                                 M_IST32_IMM(iptr->val.i, s1, disp);
2625                                 break;
2626                         case TYPE_LNG:
2627                         case TYPE_ADR:
2628                         case TYPE_DBL:
2629                                 M_IST32_IMM(iptr->val.l, s1, disp);
2630                                 M_IST32_IMM(iptr->val.l >> 32, s1, disp + 4);
2631                                 break;
2632                         }
2633                         break;
2634
2635
2636                 /* branch operations **************************************************/
2637
2638                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2639
2640                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2641                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2642
2643                         PROFILE_CYCLE_STOP;
2644
2645 #ifdef ENABLE_VERIFIER
2646                         if (iptr->val.a) {
2647                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2648                                                                         (unresolved_class *) iptr->val.a, 0);
2649
2650                                 if (opt_showdisassemble) {
2651                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2652                                 }
2653                         }
2654 #endif /* ENABLE_VERIFIER */
2655
2656                         M_CALL_IMM(0);                            /* passing exception pc */
2657                         M_POP(REG_ITMP2_XPC);
2658
2659                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2660                         M_JMP(REG_ITMP3);
2661                         break;
2662
2663                 case ICMD_GOTO:         /* ... ==> ...                                */
2664                                         /* op1 = target JavaVM pc                     */
2665
2666                         M_JMP_IMM(0);
2667                         codegen_addreference(cd, (basicblock *) iptr->target);
2668                         break;
2669
2670                 case ICMD_JSR:          /* ... ==> ...                                */
2671                                         /* op1 = target JavaVM pc                     */
2672
2673                         M_CALL_IMM(0);
2674                         codegen_addreference(cd, (basicblock *) iptr->target);
2675                         break;
2676
2677                 case ICMD_RET:          /* ... ==> ...                                */
2678                                         /* op1 = local variable                       */
2679
2680                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2681                         if (var->flags & INMEMORY) {
2682                                 M_ALD(REG_ITMP1, REG_SP, var->regoff * 8);
2683                                 M_JMP(REG_ITMP1);
2684                         } else
2685                                 M_JMP(var->regoff);
2686                         break;
2687
2688                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2689                                         /* op1 = target JavaVM pc                     */
2690
2691                         if (src->flags & INMEMORY)
2692                                 M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
2693                         else
2694                                 M_TEST(src->regoff);
2695
2696                         /* If the conditional branch is part of an if-converted
2697                            block, don't generate the actual branch. */
2698
2699                         if ((iptr->opc & ICMD_CONDITION_MASK) == 0) {
2700                                 M_BEQ(0);
2701                                 codegen_addreference(cd, (basicblock *) iptr->target);
2702                         }
2703                         break;
2704
2705                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2706                                         /* op1 = target JavaVM pc                     */
2707
2708                         if (src->flags & INMEMORY)
2709                                 M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
2710                         else
2711                                 M_TEST(src->regoff);
2712
2713                         /* If the conditional branch is part of an if-converted
2714                            block, don't generate the actual branch. */
2715
2716                         if ((iptr->opc & ICMD_CONDITION_MASK) == 0) {
2717                                 M_BNE(0);
2718                                 codegen_addreference(cd, (basicblock *) iptr->target);
2719                         }
2720                         break;
2721
2722                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2723                                         /* op1 = target JavaVM pc, val.i = constant   */
2724
2725                         emit_ifcc(cd, CC_E, src, iptr);
2726                         break;
2727
2728                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2729                                         /* op1 = target JavaVM pc, val.i = constant   */
2730
2731 #if RISC_STYLE
2732                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2733                         M_ICMP_IMM(iptr->val.i, s1);
2734                         M_BLT(0);
2735                         codegen_addreference(cd, (basicblock *) iptr->target);
2736 #else
2737                         emit_ifcc(cd, CC_L, src, iptr);
2738 #endif
2739                         break;
2740
2741                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2742                                         /* op1 = target JavaVM pc, val.i = constant   */
2743
2744                         emit_ifcc(cd, CC_LE, src, iptr);
2745                         break;
2746
2747                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2748                                         /* op1 = target JavaVM pc, val.i = constant   */
2749
2750                         emit_ifcc(cd, CC_NE, src, iptr);
2751                         break;
2752
2753                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2754                                         /* op1 = target JavaVM pc, val.i = constant   */
2755
2756                         emit_ifcc(cd, CC_G, src, iptr);
2757                         break;
2758
2759                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2760                                         /* op1 = target JavaVM pc, val.i = constant   */
2761
2762                         emit_ifcc(cd, CC_GE, src, iptr);
2763                         break;
2764
2765                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2766                                         /* op1 = target JavaVM pc, val.l = constant   */
2767
2768                         emit_if_lcc(cd, CC_E, src, iptr);
2769                         break;
2770
2771                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2772                                         /* op1 = target JavaVM pc, val.l = constant   */
2773
2774                         emit_if_lcc(cd, CC_L, src, iptr);
2775                         break;
2776
2777                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2778                                         /* op1 = target JavaVM pc, val.l = constant   */
2779
2780                         emit_if_lcc(cd, CC_LE, src, iptr);
2781                         break;
2782
2783                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2784                                         /* op1 = target JavaVM pc, val.l = constant   */
2785
2786                         emit_if_lcc(cd, CC_NE, src, iptr);
2787                         break;
2788
2789                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2790                                         /* op1 = target JavaVM pc, val.l = constant   */
2791
2792                         emit_if_lcc(cd, CC_G, src, iptr);
2793                         break;
2794
2795                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2796                                         /* op1 = target JavaVM pc, val.l = constant   */
2797
2798                         emit_if_lcc(cd, CC_GE, src, iptr);
2799                         break;
2800
2801                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2802                                         /* op1 = target JavaVM pc                     */
2803
2804                         emit_if_icmpcc(cd, CC_E, src, iptr);
2805                         break;
2806
2807                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2808                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2809
2810                         emit_if_lcmpcc(cd, CC_E, src, iptr);
2811                         break;
2812
2813                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2814                                         /* op1 = target JavaVM pc                     */
2815
2816                         emit_if_icmpcc(cd, CC_NE, src, iptr);
2817                         break;
2818
2819                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2820                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2821
2822                         emit_if_lcmpcc(cd, CC_NE, src, iptr);
2823                         break;
2824
2825                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2826                                         /* op1 = target JavaVM pc                     */
2827
2828                         emit_if_icmpcc(cd, CC_L, src, iptr);
2829                         break;
2830
2831                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2832                                     /* op1 = target JavaVM pc                     */
2833
2834                         emit_if_lcmpcc(cd, CC_L, src, iptr);
2835                         break;
2836
2837                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2838                                         /* op1 = target JavaVM pc                     */
2839
2840                         emit_if_icmpcc(cd, CC_G, src, iptr);
2841                         break;
2842
2843                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2844                                 /* op1 = target JavaVM pc                     */
2845
2846                         emit_if_lcmpcc(cd, CC_G, src, iptr);
2847                         break;
2848
2849                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2850                                         /* op1 = target JavaVM pc                     */
2851
2852                         emit_if_icmpcc(cd, CC_LE, src, iptr);
2853                         break;
2854
2855                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2856                                         /* op1 = target JavaVM pc                     */
2857
2858                         emit_if_lcmpcc(cd, CC_LE, src, iptr);
2859                         break;
2860
2861                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2862                                         /* op1 = target JavaVM pc                     */
2863
2864                         emit_if_icmpcc(cd, CC_GE, src, iptr);
2865                         break;
2866
2867                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2868                                     /* op1 = target JavaVM pc                     */
2869
2870                         emit_if_lcmpcc(cd, CC_GE, src, iptr);
2871                         break;
2872
2873                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2874
2875                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2876                         break;
2877
2878                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2879                 case ICMD_IFNE_ICONST:  /* val.i = constant                           */
2880                 case ICMD_IFLT_ICONST:
2881                 case ICMD_IFGE_ICONST:
2882                 case ICMD_IFGT_ICONST:
2883                 case ICMD_IFLE_ICONST:
2884
2885                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2886                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2887                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2888                                 if (s1 == d) {
2889                                         M_INTMOVE(s1, REG_ITMP1);
2890                                         s1 = REG_ITMP1;
2891                                 }
2892                                 if (iptr[1].val.i == 0)
2893                                         M_CLR(d);
2894                                 else
2895                                         M_IMOV_IMM(iptr[1].val.i, d);
2896                         }
2897                         if (iptr->val.i == 0)
2898                                 M_CLR(REG_ITMP2);
2899                         else
2900                                 M_IMOV_IMM(iptr->val.i, REG_ITMP2);
2901                         M_ITEST(s1);
2902
2903                         switch (iptr->opc) {
2904                         case ICMD_IFEQ_ICONST:
2905                                 M_CMOVEQ(REG_ITMP2, d);
2906                                 break;
2907                         case ICMD_IFNE_ICONST:
2908                                 M_CMOVNE(REG_ITMP2, d);
2909                                 break;
2910                         case ICMD_IFLT_ICONST:
2911                                 M_CMOVLT(REG_ITMP2, d);
2912                                 break;
2913                         case ICMD_IFGE_ICONST:
2914                                 M_CMOVGE(REG_ITMP2, d);
2915                                 break;
2916                         case ICMD_IFGT_ICONST:
2917                                 M_CMOVGT(REG_ITMP2, d);
2918                                 break;
2919                         case ICMD_IFLE_ICONST:
2920                                 M_CMOVLE(REG_ITMP2, d);
2921                                 break;
2922                         }
2923
2924                         emit_store(jd, iptr, iptr->dst, d);
2925                         break;
2926
2927
2928                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2929                 case ICMD_LRETURN:
2930
2931                         s1 = emit_load_s1(jd, iptr, src, REG_RESULT);
2932                         M_INTMOVE(s1, REG_RESULT);
2933                         goto nowperformreturn;
2934
2935                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2936
2937                         s1 = emit_load_s1(jd, iptr, src, REG_RESULT);
2938                         M_INTMOVE(s1, REG_RESULT);
2939
2940 #ifdef ENABLE_VERIFIER
2941                         if (iptr->val.a) {
2942                                 PROFILE_CYCLE_STOP;
2943
2944                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2945                                                                         (unresolved_class *) iptr->val.a, 0);
2946
2947                                 if (opt_showdisassemble) {
2948                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2949                                 }
2950
2951                                 PROFILE_CYCLE_START;
2952                         }
2953 #endif /* ENABLE_VERIFIER */
2954                         goto nowperformreturn;
2955
2956                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2957                 case ICMD_DRETURN:
2958
2959                         s1 = emit_load_s1(jd, iptr, src, REG_FRESULT);
2960                         M_FLTMOVE(s1, REG_FRESULT);
2961                         goto nowperformreturn;
2962
2963                 case ICMD_RETURN:      /* ...  ==> ...                                */
2964
2965 nowperformreturn:
2966                         {
2967                         s4 i, p;
2968
2969                         p = stackframesize;
2970
2971 #if !defined(NDEBUG)
2972                         /* generate call trace */
2973
2974                         if (opt_verbosecall) {
2975                                 emit_alu_imm_reg(cd, ALU_SUB, 2 * 8, REG_SP);
2976
2977                                 emit_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2978                                 emit_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2979
2980                                 emit_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
2981                                 emit_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2982                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2983                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2984
2985                                 M_MOV_IMM(builtin_displaymethodstop, REG_ITMP1);
2986                                 M_CALL(REG_ITMP1);
2987
2988                                 emit_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2989                                 emit_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2990
2991                                 emit_alu_imm_reg(cd, ALU_ADD, 2 * 8, REG_SP);
2992                         }
2993 #endif /* !defined(NDEBUG) */
2994
2995 #if defined(ENABLE_THREADS)
2996                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2997                                 M_ALD(rd->argintregs[0], REG_SP, rd->memuse * 8);
2998
2999                                 /* we need to save the proper return value */
3000                                 switch (iptr->opc) {
3001                                 case ICMD_IRETURN:
3002                                 case ICMD_ARETURN:
3003                                 case ICMD_LRETURN:
3004                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
3005                                         break;
3006                                 case ICMD_FRETURN:
3007                                 case ICMD_DRETURN:
3008                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
3009                                         break;
3010                                 }
3011
3012                                 M_MOV_IMM(builtin_monitorexit, REG_ITMP1);
3013                                 M_CALL(REG_ITMP1);
3014
3015                                 /* and now restore the proper return value */
3016                                 switch (iptr->opc) {
3017                                 case ICMD_IRETURN:
3018                                 case ICMD_ARETURN:
3019                                 case ICMD_LRETURN:
3020                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
3021                                         break;
3022                                 case ICMD_FRETURN:
3023                                 case ICMD_DRETURN:
3024                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
3025                                         break;
3026                                 }
3027                         }
3028 #endif
3029
3030                         /* restore saved registers */
3031
3032                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
3033                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
3034                         }
3035                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
3036                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
3037                         }
3038
3039                         /* deallocate stack */
3040
3041                         if (stackframesize)
3042                                 M_AADD_IMM(stackframesize * 8, REG_SP);
3043
3044                         /* generate method profiling code */
3045
3046                         PROFILE_CYCLE_STOP;
3047
3048                         M_RET;
3049                         }
3050                         break;
3051
3052
3053                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
3054                         {
3055                                 s4 i, l, *s4ptr;
3056                                 void **tptr;
3057
3058                                 tptr = (void **) iptr->target;
3059
3060                                 s4ptr = iptr->val.a;
3061                                 l = s4ptr[1];                          /* low     */
3062                                 i = s4ptr[2];                          /* high    */
3063
3064                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3065                                 M_INTMOVE(s1, REG_ITMP1);
3066                                 if (l != 0) {
3067                                         emit_alul_imm_reg(cd, ALU_SUB, l, REG_ITMP1);
3068                                 }
3069                                 i = i - l + 1;
3070
3071                 /* range check */
3072                                 emit_alul_imm_reg(cd, ALU_CMP, i - 1, REG_ITMP1);
3073                                 emit_jcc(cd, CC_A, 0);
3074
3075                                 codegen_addreference(cd, (basicblock *) tptr[0]);
3076
3077                                 /* build jump table top down and use address of lowest entry */
3078
3079                 /* s4ptr += 3 + i; */
3080                                 tptr += i;
3081
3082                                 while (--i >= 0) {
3083                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
3084                                         --tptr;
3085                                 }
3086
3087                                 /* length of dataseg after last dseg_addtarget is used
3088                                    by load */
3089
3090                                 M_MOV_IMM(0, REG_ITMP2);
3091                                 dseg_adddata(cd);
3092                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
3093                                 emit_jmp_reg(cd, REG_ITMP1);
3094                         }
3095                         break;
3096
3097
3098                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
3099                         {
3100                                 s4 i, l, val, *s4ptr;
3101                                 void **tptr;
3102
3103                                 tptr = (void **) iptr->target;
3104
3105                                 s4ptr = iptr->val.a;
3106                                 l = s4ptr[0];                          /* default  */
3107                                 i = s4ptr[1];                          /* count    */
3108
3109                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
3110                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);    /* reg compare should always be faster */
3111                                 while (--i >= 0) {
3112                                         s4ptr += 2;
3113                                         ++tptr;
3114
3115                                         val = s4ptr[0];
3116                                         emit_alul_imm_reg(cd, ALU_CMP, val, s1);
3117                                         emit_jcc(cd, CC_E, 0);
3118                                         codegen_addreference(cd, (basicblock *) tptr[0]);
3119                                 }
3120
3121                                 emit_jmp_imm(cd, 0);
3122
3123                                 tptr = (void **) iptr->target;
3124                                 codegen_addreference(cd, (basicblock *) tptr[0]);
3125                         }
3126                         break;
3127
3128
3129                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
3130                                         /* op1 = arg count val.a = builtintable entry */
3131
3132                         bte = iptr->val.a;
3133                         md = bte->md;
3134                         goto gen_method;
3135
3136                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
3137                                         /* op1 = arg count, val.a = method pointer    */
3138
3139                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
3140                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
3141                 case ICMD_INVOKEINTERFACE:
3142
3143                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3144                                 md = INSTRUCTION_UNRESOLVED_METHOD(iptr)->methodref->parseddesc.md;
3145                                 lm = NULL;
3146                         }
3147                         else {
3148                                 lm = INSTRUCTION_RESOLVED_METHODINFO(iptr);
3149                                 md = lm->parseddesc;
3150                         }
3151
3152 gen_method:
3153                         s3 = md->paramcount;
3154
3155                         MCODECHECK((20 * s3) + 128);
3156
3157                         /* copy arguments to registers or stack location */
3158
3159                         for (s3 = s3 - 1; s3 >= 0; s3--, src = src->prev) {
3160                                 if (src->varkind == ARGVAR)
3161                                         continue;
3162                                 if (IS_INT_LNG_TYPE(src->type)) {
3163                                         if (!md->params[s3].inmemory) {
3164                                                 s1 = rd->argintregs[md->params[s3].regoff];
3165                                                 d = emit_load_s1(jd, iptr, src, s1);
3166                                                 M_INTMOVE(d, s1);
3167                                         } else {
3168                                                 d = emit_load_s1(jd, iptr, src, REG_ITMP1);
3169                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
3170                                         }
3171
3172                                 } else {
3173                                         if (!md->params[s3].inmemory) {
3174                                                 s1 = rd->argfltregs[md->params[s3].regoff];
3175                                                 d = emit_load_s1(jd, iptr, src, s1);
3176                                                 M_FLTMOVE(d, s1);
3177                                         } else {
3178                                                 d = emit_load_s1(jd, iptr, src, REG_FTMP1);
3179                                                 M_DST(d, REG_SP, md->params[s3].regoff * 8);
3180                                         }
3181                                 }
3182                         }
3183
3184                         /* generate method profiling code */
3185
3186                         PROFILE_CYCLE_STOP;
3187
3188                         switch (iptr->opc) {
3189                         case ICMD_BUILTIN:
3190                                 a = (ptrint) bte->fp;
3191                                 d = md->returntype.type;
3192
3193                                 M_MOV_IMM(a, REG_ITMP1);
3194                                 M_CALL(REG_ITMP1);
3195
3196                                 /* if op1 == true, we need to check for an exception */
3197
3198                                 if (iptr->op1 == true) {
3199                                         M_TEST(REG_RESULT);
3200                                         M_BEQ(0);
3201                                         codegen_add_fillinstacktrace_ref(cd);
3202                                 }
3203                                 break;
3204
3205                         case ICMD_INVOKESPECIAL:
3206                                 M_TEST(rd->argintregs[0]);
3207                                 M_BEQ(0);
3208                                 codegen_add_nullpointerexception_ref(cd);
3209
3210                                 /* first argument contains pointer */
3211 /*                              gen_nullptr_check(rd->argintregs[0]); */
3212
3213                                 /* access memory for hardware nullptr */
3214 /*                              emit_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); */
3215
3216                                 /* fall through */
3217
3218                         case ICMD_INVOKESTATIC:
3219                                 if (lm == NULL) {
3220                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3221
3222                                         codegen_addpatchref(cd, PATCHER_invokestatic_special,
3223                                                                                 um, 0);
3224
3225                                         if (opt_showdisassemble) {
3226                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3227                                         }
3228
3229                                         a = 0;
3230                                         d = um->methodref->parseddesc.md->returntype.type;
3231
3232                                 } else {
3233                                         a = (ptrint) lm->stubroutine;
3234                                         d = lm->parseddesc->returntype.type;
3235                                 }
3236
3237                                 M_MOV_IMM(a, REG_ITMP2);
3238                                 M_CALL(REG_ITMP2);
3239                                 break;
3240
3241                         case ICMD_INVOKEVIRTUAL:
3242                                 gen_nullptr_check(rd->argintregs[0]);
3243
3244                                 if (lm == NULL) {
3245                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3246
3247                                         codegen_addpatchref(cd, PATCHER_invokevirtual, um, 0);
3248
3249                                         if (opt_showdisassemble) {
3250                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3251                                         }
3252
3253                                         s1 = 0;
3254                                         d = um->methodref->parseddesc.md->returntype.type;
3255
3256                                 } else {
3257                                         s1 = OFFSET(vftbl_t, table[0]) +
3258                                                 sizeof(methodptr) * lm->vftblindex;
3259                                         d = lm->parseddesc->returntype.type;
3260                                 }
3261
3262                                 M_ALD(REG_METHODPTR, rd->argintregs[0],
3263                                           OFFSET(java_objectheader, vftbl));
3264                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
3265                                 M_CALL(REG_ITMP3);
3266                                 break;
3267
3268                         case ICMD_INVOKEINTERFACE:
3269                                 gen_nullptr_check(rd->argintregs[0]);
3270
3271                                 if (lm == NULL) {
3272                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3273
3274                                         codegen_addpatchref(cd, PATCHER_invokeinterface, um, 0);
3275
3276                                         if (opt_showdisassemble) {
3277                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3278                                         }
3279
3280                                         s1 = 0;
3281                                         s2 = 0;
3282                                         d = um->methodref->parseddesc.md->returntype.type;
3283
3284                                 } else {
3285                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3286                                                 sizeof(methodptr) * lm->class->index;
3287
3288                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3289
3290                                         d = lm->parseddesc->returntype.type;
3291                                 }
3292
3293                                 M_ALD(REG_METHODPTR, rd->argintregs[0],
3294                                           OFFSET(java_objectheader, vftbl));
3295                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
3296                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
3297                                 M_CALL(REG_ITMP3);
3298                                 break;
3299                         }
3300
3301                         /* generate method profiling code */
3302
3303                         PROFILE_CYCLE_START;
3304
3305                         /* d contains return type */
3306
3307                         if (d != TYPE_VOID) {
3308                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3309                                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_RESULT);
3310                                         M_INTMOVE(REG_RESULT, s1);
3311                                         emit_store(jd, iptr, iptr->dst, s1);
3312                                 } else {
3313                                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FRESULT);
3314                                         M_FLTMOVE(REG_FRESULT, s1);
3315                                         emit_store(jd, iptr, iptr->dst, s1);
3316                                 }
3317                         }
3318                         break;
3319
3320
3321                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3322
3323                                       /* op1:   0 == array, 1 == class                */
3324                                       /* val.a: (classinfo *) superclass              */
3325
3326                         /*  superclass is an interface:
3327                          *
3328                          *  OK if ((sub == NULL) ||
3329                          *         (sub->vftbl->interfacetablelength > super->index) &&
3330                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3331                          *
3332                          *  superclass is a class:
3333                          *
3334                          *  OK if ((sub == NULL) || (0
3335                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3336                          *         super->vftbl->diffval));
3337                          */
3338
3339                         if (iptr->op1 == 1) {
3340                                 /* object type cast-check */
3341
3342                                 classinfo *super;
3343                                 vftbl_t   *supervftbl;
3344                                 s4         superindex;
3345
3346                                 super = (classinfo *) iptr->val.a;
3347
3348                                 if (!super) {
3349                                         superindex = 0;
3350                                         supervftbl = NULL;
3351
3352                                 } else {
3353                                         superindex = super->index;
3354                                         supervftbl = super->vftbl;
3355                                 }
3356
3357 #if defined(ENABLE_THREADS)
3358                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3359 #endif
3360                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3361
3362                                 /* calculate interface checkcast code size */
3363
3364                                 s2 = 3; /* mov_membase_reg */
3365                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3366
3367                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3368                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3369                                         3 /* test */ + 6 /* jcc */;
3370
3371                                 if (!super)
3372                                         s2 += (opt_showdisassemble ? 5 : 0);
3373
3374                                 /* calculate class checkcast code size */
3375
3376                                 s3 = 3; /* mov_membase_reg */
3377                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3378                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3379
3380 #if 0
3381                                 if (s1 != REG_ITMP1) {
3382                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3383                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3384                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3385                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3386                                         a += 3;    /* sub */
3387
3388                                 } else
3389 #endif
3390                                         {
3391                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3392                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3393                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3394                                         }
3395
3396                                 s3 += 3 /* cmp */ + 6 /* jcc */;
3397
3398                                 if (!super)
3399                                         s3 += (opt_showdisassemble ? 5 : 0);
3400
3401                                 /* if class is not resolved, check which code to call */
3402
3403                                 if (!super) {
3404                                         M_TEST(s1);
3405                                         M_BEQ(6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3406
3407                                         codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3408                                                                                 (constant_classref *) iptr->target, 0);
3409
3410                                         if (opt_showdisassemble) {
3411                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3412                                         }
3413
3414                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
3415                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
3416                                         M_BEQ(s2 + 5);
3417                                 }
3418
3419                                 /* interface checkcast code */
3420
3421                                 if (!super || (super->flags & ACC_INTERFACE)) {
3422                                         if (super) {
3423                                                 M_TEST(s1);
3424                                                 M_BEQ(s2);
3425                                         }
3426
3427                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3428
3429                                         if (!super) {
3430                                                 codegen_addpatchref(cd,
3431                                                                                         PATCHER_checkcast_instanceof_interface,
3432                                                                                         (constant_classref *) iptr->target,
3433                                                                                         0);
3434
3435                                                 if (opt_showdisassemble) {
3436                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3437                                                 }
3438                                         }
3439
3440                                         emit_movl_membase32_reg(cd, REG_ITMP2,
3441                                                                                           OFFSET(vftbl_t, interfacetablelength),
3442                                                                                           REG_ITMP3);
3443                                         /* XXX TWISTI: should this be int arithmetic? */
3444                                         M_LSUB_IMM32(superindex, REG_ITMP3);
3445                                         M_TEST(REG_ITMP3);
3446                                         M_BLE(0);
3447                                         codegen_add_classcastexception_ref(cd);
3448                                         emit_mov_membase32_reg(cd, REG_ITMP2,
3449                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
3450                                                                                          superindex * sizeof(methodptr*),
3451                                                                                          REG_ITMP3);
3452                                         M_TEST(REG_ITMP3);
3453                                         M_BEQ(0);
3454                                         codegen_add_classcastexception_ref(cd);
3455
3456                                         if (!super)
3457                                                 M_JMP_IMM(s3);
3458                                 }
3459
3460                                 /* class checkcast code */
3461
3462                                 if (!super || !(super->flags & ACC_INTERFACE)) {
3463                                         if (super) {
3464                                                 M_TEST(s1);
3465                                                 M_BEQ(s3);
3466                                         }
3467
3468                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3469
3470                                         if (!super) {
3471                                                 codegen_addpatchref(cd, PATCHER_checkcast_class,
3472                                                                                         (constant_classref *) iptr->target,
3473                                                                                         0);
3474
3475                                                 if (opt_showdisassemble) {
3476                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3477                                                 }
3478                                         }
3479
3480                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3481 #if defined(ENABLE_THREADS)
3482                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3483 #endif
3484                                         emit_movl_membase32_reg(cd, REG_ITMP2,
3485                                                                                           OFFSET(vftbl_t, baseval),
3486                                                                                           REG_ITMP2);
3487                                         /*                                      if (s1 != REG_ITMP1) { */
3488                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3489                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
3490                                         /*                                                                                              REG_ITMP1); */
3491                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3492                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3493                                         /*                                                                                              REG_ITMP3); */
3494                                         /*  #if defined(ENABLE_THREADS) */
3495                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3496                                         /*  #endif */
3497                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
3498
3499                                         /*                                      } else { */
3500                                         emit_movl_membase32_reg(cd, REG_ITMP3,
3501                                                                                           OFFSET(vftbl_t, baseval),
3502                                                                                           REG_ITMP3);
3503                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3504                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3505                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3506                                         /*                                      } */
3507 #if defined(ENABLE_THREADS)
3508                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3509 #endif
3510                                         M_CMP(REG_ITMP3, REG_ITMP2);
3511                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3512                                         codegen_add_classcastexception_ref(cd);
3513                                 }
3514                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
3515
3516                         } else {
3517                                 /* array type cast-check */
3518
3519                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3520                                 M_INTMOVE(s1, rd->argintregs[0]);
3521
3522                                 if (iptr->val.a == NULL) {
3523                                         codegen_addpatchref(cd, PATCHER_builtin_arraycheckcast,
3524                                                                                 (constant_classref *) iptr->target, 0);
3525
3526                                         if (opt_showdisassemble) {
3527                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3528                                         }
3529                                 }
3530
3531                                 M_MOV_IMM(iptr->val.a, rd->argintregs[1]);
3532                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
3533                                 M_CALL(REG_ITMP1);
3534                                 M_TEST(REG_RESULT);
3535                                 M_BEQ(0);
3536                                 codegen_add_classcastexception_ref(cd);
3537
3538                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3539                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
3540                         }
3541                         M_INTMOVE(s1, d);
3542                         emit_store(jd, iptr, iptr->dst, d);
3543                         break;
3544
3545                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3546
3547                                       /* op1:   0 == array, 1 == class                */
3548                                       /* val.a: (classinfo *) superclass              */
3549
3550                         /*  superclass is an interface:
3551                          *
3552                          *  return (sub != NULL) &&
3553                          *         (sub->vftbl->interfacetablelength > super->index) &&
3554                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3555                          *
3556                          *  superclass is a class:
3557                          *
3558                          *  return ((sub != NULL) && (0
3559                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3560                          *          super->vftbl->diffvall));
3561                          */
3562
3563                         {
3564                         classinfo *super;
3565                         vftbl_t   *supervftbl;
3566                         s4         superindex;
3567
3568                         super = (classinfo *) iptr->val.a;
3569
3570                         if (!super) {
3571                                 superindex = 0;
3572                                 supervftbl = NULL;
3573
3574                         } else {
3575                                 superindex = super->index;
3576                                 supervftbl = super->vftbl;
3577                         }
3578
3579 #if defined(ENABLE_THREADS)
3580             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3581 #endif
3582
3583                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3584                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
3585                         if (s1 == d) {
3586                                 M_INTMOVE(s1, REG_ITMP1);
3587                                 s1 = REG_ITMP1;
3588                         }
3589
3590                         /* calculate interface instanceof code size */
3591
3592                         s2 = 3; /* mov_membase_reg */
3593                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3594                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3595                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3596                                 3 /* test */ + 4 /* setcc */;
3597
3598                         if (!super)
3599                                 s2 += (opt_showdisassemble ? 5 : 0);
3600
3601                         /* calculate class instanceof code size */
3602
3603                         s3 = 3; /* mov_membase_reg */
3604                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3605                         s3 += 10; /* mov_imm_reg */
3606                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3607                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3608                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3609                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3610                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3611                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3612                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3613
3614                         if (!super)
3615                                 s3 += (opt_showdisassemble ? 5 : 0);
3616
3617                         emit_alu_reg_reg(cd, ALU_XOR, d, d);
3618
3619                         /* if class is not resolved, check which code to call */
3620
3621                         if (!super) {
3622                                 emit_test_reg_reg(cd, s1, s1);
3623                                 emit_jcc(cd, CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3624                                                                                          7 + 6 + s2 + 5 + s3));
3625
3626                                 codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3627                                                                         (constant_classref *) iptr->target, 0);
3628
3629                                 if (opt_showdisassemble) {
3630                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3631                                 }
3632
3633                                 emit_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3634                                 emit_alul_imm_reg(cd, ALU_AND, ACC_INTERFACE, REG_ITMP3);
3635                                 emit_jcc(cd, CC_Z, s2 + 5);
3636                         }
3637
3638                         /* interface instanceof code */
3639
3640                         if (!super || (super->flags & ACC_INTERFACE)) {
3641                                 if (super) {
3642                                         emit_test_reg_reg(cd, s1, s1);
3643                                         emit_jcc(cd, CC_Z, s2);
3644                                 }
3645
3646                                 emit_mov_membase_reg(cd, s1,
3647                                                                            OFFSET(java_objectheader, vftbl),
3648                                                                            REG_ITMP1);
3649                                 if (!super) {
3650                                         codegen_addpatchref(cd,
3651                                                                                 PATCHER_checkcast_instanceof_interface,
3652                                                                                 (constant_classref *) iptr->target, 0);
3653
3654                                         if (opt_showdisassemble) {
3655                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3656                                         }
3657                                 }
3658
3659                                 emit_movl_membase32_reg(cd, REG_ITMP1,
3660                                                                                   OFFSET(vftbl_t, interfacetablelength),
3661                                                                                   REG_ITMP3);
3662                                 emit_alu_imm32_reg(cd, ALU_SUB, superindex, REG_ITMP3);
3663                                 emit_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3664
3665                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3666
3667                                 emit_jcc(cd, CC_LE, a);
3668                                 emit_mov_membase32_reg(cd, REG_ITMP1,
3669                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3670                                                                                  superindex * sizeof(methodptr*),
3671                                                                                  REG_ITMP1);
3672                                 emit_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3673                                 emit_setcc_reg(cd, CC_NE, d);
3674
3675                                 if (!super)
3676                                         emit_jmp_imm(cd, s3);
3677                         }
3678
3679                         /* class instanceof code */
3680
3681                         if (!super || !(super->flags & ACC_INTERFACE)) {
3682                                 if (super) {
3683                                         emit_test_reg_reg(cd, s1, s1);
3684                                         emit_jcc(cd, CC_E, s3);
3685                                 }
3686
3687                                 emit_mov_membase_reg(cd, s1,
3688                                                                            OFFSET(java_objectheader, vftbl),
3689                                                                            REG_ITMP1);
3690
3691                                 if (!super) {
3692                                         codegen_addpatchref(cd, PATCHER_instanceof_class,
3693                                                                                 (constant_classref *) iptr->target, 0);
3694
3695                                         if (opt_showdisassemble) {
3696                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3697                                         }
3698                                 }
3699
3700                                 emit_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3701 #if defined(ENABLE_THREADS)
3702                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3703 #endif
3704                                 emit_movl_membase_reg(cd, REG_ITMP1,
3705                                                                                 OFFSET(vftbl_t, baseval),
3706                                                                                 REG_ITMP1);
3707                                 emit_movl_membase_reg(cd, REG_ITMP2,
3708                                                                                 OFFSET(vftbl_t, diffval),
3709                                                                                 REG_ITMP3);
3710                                 emit_movl_membase_reg(cd, REG_ITMP2,
3711                                                                                 OFFSET(vftbl_t, baseval),
3712                                                                                 REG_ITMP2);
3713 #if defined(ENABLE_THREADS)
3714                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3715 #endif
3716                                 emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
3717                                 emit_alu_reg_reg(cd, ALU_XOR, d, d); /* may be REG_ITMP2 */
3718                                 emit_alu_reg_reg(cd, ALU_CMP, REG_ITMP3, REG_ITMP1);
3719                                 emit_setcc_reg(cd, CC_BE, d);
3720                         }
3721                         emit_store(jd, iptr, iptr->dst, d);
3722                         }
3723                         break;
3724
3725                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3726                                          /* op1 = dimension, val.a = class            */
3727
3728                         /* check for negative sizes and copy sizes to stack if necessary  */
3729
3730                         MCODECHECK((10 * 4 * iptr->op1) + 5 + 10 * 8);
3731
3732                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3733                                 /* copy SAVEDVAR sizes to stack */
3734
3735                                 if (src->varkind != ARGVAR) {
3736                                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP1);
3737                                         M_LST(s2, REG_SP, s1 * 8);
3738                                 }
3739                         }
3740
3741                         /* is a patcher function set? */
3742
3743                         if (iptr->val.a == NULL) {
3744                                 codegen_addpatchref(cd, PATCHER_builtin_multianewarray,
3745                                                                         (constant_classref *) iptr->target, 0);
3746
3747                                 if (opt_showdisassemble) {
3748                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3749                                 }
3750
3751                                 a = 0;
3752
3753                         } else {
3754                                 a = (ptrint) iptr->val.a;
3755                         }
3756
3757                         /* a0 = dimension count */
3758
3759                         M_MOV_IMM(iptr->op1, rd->argintregs[0]);
3760
3761                         /* a1 = arrayvftbl */
3762
3763                         M_MOV_IMM(iptr->val.a, rd->argintregs[1]);
3764
3765                         /* a2 = pointer to dimensions = stack pointer */
3766
3767                         M_MOV(REG_SP, rd->argintregs[2]);
3768
3769                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3770                         M_CALL(REG_ITMP1);
3771
3772                         /* check for exception before result assignment */
3773
3774                         M_TEST(REG_RESULT);
3775                         M_BEQ(0);
3776                         codegen_add_fillinstacktrace_ref(cd);
3777
3778                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_RESULT);
3779                         M_INTMOVE(REG_RESULT, s1);
3780                         emit_store(jd, iptr, iptr->dst, s1);
3781                         break;
3782
3783                 default:
3784                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3785                         return false;
3786         } /* switch */
3787
3788         } /* for instruction */
3789
3790         /* copy values to interface registers */
3791
3792         src = bptr->outstack;
3793         len = bptr->outdepth;
3794         MCODECHECK(512);
3795 #if defined(ENABLE_LSRA)
3796         if (!opt_lsra)
3797 #endif
3798         while (src) {
3799                 len--;
3800                 if ((src->varkind != STACKVAR)) {
3801                         s2 = src->type;
3802                         if (IS_FLT_DBL_TYPE(s2)) {
3803                                 s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
3804                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3805                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3806
3807                                 } else {
3808                                         emit_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3809                                 }
3810
3811                         } else {
3812                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3813                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3814                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3815
3816                                 } else {
3817                                         emit_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3818                                 }
3819                         }
3820                 }
3821                 src = src->prev;
3822         }
3823
3824         /* At the end of a basic block we may have to append some nops,
3825            because the patcher stub calling code might be longer than the
3826            actual instruction. So codepatching does not change the
3827            following block unintentionally. */
3828
3829         if (cd->mcodeptr < cd->lastmcodeptr) {
3830                 while (cd->mcodeptr < cd->lastmcodeptr) {
3831                         M_NOP;
3832                 }
3833         }
3834
3835         } /* if (bptr -> flags >= BBREACHED) */
3836         } /* for basic block */
3837
3838         dseg_createlinenumbertable(cd);
3839
3840
3841         /* generate exception and patcher stubs */
3842
3843         {
3844                 exceptionref *eref;
3845                 patchref     *pref;
3846                 ptrint        mcode;
3847                 u1           *savedmcodeptr;
3848                 u1           *tmpmcodeptr;
3849
3850                 savedmcodeptr = NULL;
3851
3852                 /* generate exception stubs */
3853
3854                 for (eref = cd->exceptionrefs; eref != NULL; eref = eref->next) {
3855                         gen_resolvebranch(cd->mcodebase + eref->branchpos,
3856                                                           eref->branchpos,
3857                                                           cd->mcodeptr - cd->mcodebase);
3858
3859                         MCODECHECK(512);
3860
3861                         /* Check if the exception is an
3862                            ArrayIndexOutOfBoundsException.  If so, move index register
3863                            into a4. */
3864
3865                         if (eref->reg != -1)
3866                                 M_MOV(eref->reg, rd->argintregs[4]);
3867
3868                         /* calcuate exception address */
3869
3870                         M_MOV_IMM(0, rd->argintregs[3]);
3871                         dseg_adddata(cd);
3872                         M_AADD_IMM32(eref->branchpos - 6, rd->argintregs[3]);
3873
3874                         /* move function to call into REG_ITMP3 */
3875
3876                         M_MOV_IMM(eref->function, REG_ITMP3);
3877
3878                         if (savedmcodeptr != NULL) {
3879                                 M_JMP_IMM(savedmcodeptr - cd->mcodeptr - 5);
3880
3881                         } else {
3882                                 savedmcodeptr = cd->mcodeptr;
3883
3884                                 emit_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3885                                 M_MOV(REG_SP, rd->argintregs[1]);
3886                                 M_ALD(rd->argintregs[2], REG_SP, stackframesize * 8);
3887
3888                                 M_ASUB_IMM(2 * 8, REG_SP);
3889                                 M_AST(rd->argintregs[3], REG_SP, 0 * 8);         /* store XPC */
3890
3891                                 M_CALL(REG_ITMP3);
3892
3893                                 M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3894                                 M_AADD_IMM(2 * 8, REG_SP);
3895
3896                                 M_MOV_IMM(asm_handle_exception, REG_ITMP3);
3897                                 M_JMP(REG_ITMP3);
3898                         }
3899                 }
3900
3901
3902                 /* generate code patching stub call code */
3903
3904                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
3905                         /* check size of code segment */
3906
3907                         MCODECHECK(512);
3908
3909                         /* Get machine code which is patched back in later. A
3910                            `call rel32' is 5 bytes long (but read 8 bytes). */
3911
3912                         savedmcodeptr = cd->mcodebase + pref->branchpos;
3913                         mcode = *((ptrint *) savedmcodeptr);
3914
3915                         /* patch in `call rel32' to call the following code */
3916
3917                         tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr          */
3918                         cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position */
3919
3920                         M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
3921
3922                         cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr   */
3923
3924                         /* move pointer to java_objectheader onto stack */
3925
3926 #if defined(ENABLE_THREADS)
3927                         /* create a virtual java_objectheader */
3928
3929                         (void) dseg_addaddress(cd, NULL);                         /* flcword    */
3930                         (void) dseg_addaddress(cd, lock_get_initial_lock_word()); /* monitorPtr */
3931                         a = dseg_addaddress(cd, NULL);                            /* vftbl      */
3932
3933                         emit_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP3);
3934                         M_PUSH(REG_ITMP3);
3935 #else
3936                         M_PUSH_IMM(0);
3937 #endif
3938
3939                         /* move machine code bytes and classinfo pointer into registers */
3940
3941                         M_MOV_IMM(mcode, REG_ITMP3);
3942                         M_PUSH(REG_ITMP3);
3943                         M_MOV_IMM(pref->ref, REG_ITMP3);
3944                         M_PUSH(REG_ITMP3);
3945                         M_MOV_IMM(pref->disp, REG_ITMP3);
3946                         M_PUSH(REG_ITMP3);
3947
3948                         M_MOV_IMM(pref->patcher, REG_ITMP3);
3949                         M_PUSH(REG_ITMP3);
3950
3951                         M_MOV_IMM(asm_wrapper_patcher, REG_ITMP3);
3952                         M_JMP(REG_ITMP3);
3953                 }
3954         }
3955
3956         /* generate replacement-out stubs */
3957
3958         {
3959                 int i;
3960
3961                 replacementpoint = jd->code->rplpoints;
3962
3963                 for (i = 0; i < jd->code->rplpointcount; ++i, ++replacementpoint) {
3964                         /* check code segment size */
3965
3966                         MCODECHECK(512);
3967
3968                         /* note start of stub code */
3969
3970                         replacementpoint->outcode = (u1*) (ptrint)(cd->mcodeptr - cd->mcodebase);
3971
3972                         /* make machine code for patching */
3973
3974                         disp = (ptrint)(replacementpoint->outcode - replacementpoint->pc) - 5;
3975                         replacementpoint->mcode = 0xe9 | ((u8)disp << 8);
3976
3977                         /* push address of `rplpoint` struct */
3978
3979                         M_MOV_IMM(replacementpoint, REG_ITMP3);
3980                         M_PUSH(REG_ITMP3);
3981
3982                         /* jump to replacement function */
3983
3984                         M_MOV_IMM(asm_replacement_out, REG_ITMP3);
3985                         M_JMP(REG_ITMP3);
3986                 }
3987         }
3988
3989         codegen_finish(jd);
3990
3991         /* everything's ok */
3992
3993         return true;
3994 }
3995
3996
3997 /* createcompilerstub **********************************************************
3998
3999    Creates a stub routine which calls the compiler.
4000
4001 *******************************************************************************/
4002
4003 #define COMPILERSTUB_DATASIZE    2 * SIZEOF_VOID_P
4004 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
4005
4006 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
4007
4008
4009 u1 *createcompilerstub(methodinfo *m)
4010 {
4011         u1          *s;                     /* memory to hold the stub            */
4012         ptrint      *d;
4013         codegendata *cd;
4014         s4           dumpsize;
4015
4016         s = CNEW(u1, COMPILERSTUB_SIZE);
4017
4018         /* set data pointer and code pointer */
4019
4020         d = (ptrint *) s;
4021         s = s + COMPILERSTUB_DATASIZE;
4022
4023         /* mark start of dump memory area */
4024
4025         dumpsize = dump_size();
4026
4027         cd = DNEW(codegendata);
4028         cd->mcodeptr = s;
4029
4030         /* Store the methodinfo* in the same place as in the methodheader
4031            for compiled methods. */
4032
4033         d[0] = (ptrint) asm_call_jit_compiler;
4034         d[1] = (ptrint) m;
4035
4036         /* code for the stub */
4037
4038         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 1 * SIZEOF_VOID_P));       /* methodinfo  */
4039         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 2 * SIZEOF_VOID_P));  /* compiler pointer */
4040         M_JMP(REG_ITMP3);
4041
4042 #if defined(ENABLE_STATISTICS)
4043         if (opt_stat)
4044                 count_cstub_len += COMPILERSTUB_SIZE;
4045 #endif
4046
4047         /* release dump area */
4048
4049         dump_release(dumpsize);
4050
4051         return s;
4052 }
4053
4054
4055 /* createnativestub ************************************************************
4056
4057    Creates a stub routine which calls a native method.
4058
4059 *******************************************************************************/
4060
4061 u1 *createnativestub(functionptr f, jitdata *jd, methoddesc *nmd)
4062 {
4063         methodinfo   *m;
4064         codegendata  *cd;
4065         registerdata *rd;
4066         methoddesc   *md;
4067         s4            stackframesize;       /* size of stackframe if needed       */
4068         s4            nativeparams;
4069         s4            i, j;                 /* count variables                    */
4070         s4            t;
4071         s4            s1, s2;
4072
4073         /* get required compiler data */
4074
4075         m  = jd->m;
4076         cd = jd->cd;
4077         rd = jd->rd;
4078
4079         /* initialize variables */
4080
4081         md = m->parseddesc;
4082         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
4083
4084         /* calculate stack frame size */
4085
4086         stackframesize =
4087                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
4088                 sizeof(localref_table) / SIZEOF_VOID_P +
4089                 INT_ARG_CNT + FLT_ARG_CNT + 1 +         /* + 1 for function address   */
4090                 nmd->memuse;
4091
4092         if (!(stackframesize & 0x1))                /* keep stack 16-byte aligned */
4093                 stackframesize++;
4094
4095         /* create method header */
4096
4097         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
4098         (void) dseg_adds4(cd, stackframesize * 8);              /* FrameSize      */
4099         (void) dseg_adds4(cd, 0);                               /* IsSync         */
4100         (void) dseg_adds4(cd, 0);                               /* IsLeaf         */
4101         (void) dseg_adds4(cd, 0);                               /* IntSave        */
4102         (void) dseg_adds4(cd, 0);                               /* FltSave        */
4103         (void) dseg_addlinenumbertablesize(cd);
4104         (void) dseg_adds4(cd, 0);                               /* ExTableSize    */
4105
4106         /* generate native method profiling code */
4107
4108         if (opt_prof) {
4109                 /* count frequency */
4110
4111                 M_MOV_IMM(m, REG_ITMP2);
4112                 M_IINC_MEMBASE(REG_ITMP2, OFFSET(methodinfo, frequency));
4113         }
4114
4115         /* generate stub code */
4116
4117         M_ASUB_IMM(stackframesize * 8, REG_SP);
4118
4119 #if !defined(NDEBUG)
4120         /* generate call trace */
4121
4122         if (opt_verbosecall) {
4123                 /* save integer and float argument registers */
4124
4125                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4126                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4127                                 M_LST(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4128
4129                 for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4130                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4131                                 M_DST(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4132
4133                 /* show integer hex code for float arguments */
4134
4135                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++) {
4136                         /* if the paramtype is a float, we have to right shift all
4137                            following integer registers */
4138
4139                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type)) {
4140                                 for (s1 = INT_ARG_CNT - 2; s1 >= i; s1--)
4141                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
4142
4143                                 emit_movd_freg_reg(cd, rd->argfltregs[j], rd->argintregs[i]);
4144                                 j++;
4145                         }
4146                 }
4147
4148                 M_MOV_IMM(m, REG_ITMP1);
4149                 M_AST(REG_ITMP1, REG_SP, 0 * 8);
4150                 M_MOV_IMM(builtin_trace_args, REG_ITMP1);
4151                 M_CALL(REG_ITMP1);
4152
4153                 /* restore integer and float argument registers */
4154
4155                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4156                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4157                                 M_LLD(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4158
4159                 for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4160                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4161                                 M_DLD(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4162         }
4163 #endif /* !defined(NDEBUG) */
4164
4165         /* get function address (this must happen before the stackframeinfo) */
4166
4167 #if !defined(WITH_STATIC_CLASSPATH)
4168         if (f == NULL) {
4169                 codegen_addpatchref(cd, PATCHER_resolve_native, m, 0);
4170
4171                 if (opt_showdisassemble) {
4172                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
4173                 }
4174         }
4175 #endif
4176
4177         M_MOV_IMM(f, REG_ITMP3);
4178
4179
4180         /* save integer and float argument registers */
4181
4182         for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4183                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4184                         M_LST(rd->argintregs[j++], REG_SP, i * 8);
4185
4186         for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4187                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4188                         M_DST(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4189
4190         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4191
4192         /* create dynamic stack info */
4193
4194         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4195         emit_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[1]);
4196         M_ALEA(REG_SP, stackframesize * 8 + SIZEOF_VOID_P, rd->argintregs[2]);
4197         M_ALD(rd->argintregs[3], REG_SP, stackframesize * 8);
4198         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
4199         M_CALL(REG_ITMP1);
4200
4201         /* restore integer and float argument registers */
4202
4203         for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4204                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4205                         M_LLD(rd->argintregs[j++], REG_SP, i * 8);
4206
4207         for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4208                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4209                         M_DLD(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4210
4211         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4212
4213
4214         /* copy or spill arguments to new locations */
4215
4216         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
4217                 t = md->paramtypes[i].type;
4218
4219                 if (IS_INT_LNG_TYPE(t)) {
4220                         if (!md->params[i].inmemory) {
4221                                 s1 = rd->argintregs[md->params[i].regoff];
4222
4223                                 if (!nmd->params[j].inmemory) {
4224                                         s2 = rd->argintregs[nmd->params[j].regoff];
4225                                         M_INTMOVE(s1, s2);
4226
4227                                 } else {
4228                                         s2 = nmd->params[j].regoff;
4229                                         M_LST(s1, REG_SP, s2 * 8);
4230                                 }
4231
4232                         } else {
4233                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4234                                 s2 = nmd->params[j].regoff;
4235                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
4236                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
4237                         }
4238
4239                 } else {
4240                         /* We only copy spilled float arguments, as the float argument    */
4241                         /* registers keep unchanged.                                      */
4242
4243                         if (md->params[i].inmemory) {
4244                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4245                                 s2 = nmd->params[j].regoff;
4246                                 M_DLD(REG_FTMP1, REG_SP, s1 * 8);
4247                                 M_DST(REG_FTMP1, REG_SP, s2 * 8);
4248                         }
4249                 }
4250         }
4251
4252         /* put class into second argument register */
4253
4254         if (m->flags & ACC_STATIC)
4255                 M_MOV_IMM(m->class, rd->argintregs[1]);
4256
4257         /* put env into first argument register */
4258
4259         M_MOV_IMM(_Jv_env, rd->argintregs[0]);
4260
4261         /* do the native function call */
4262
4263         M_CALL(REG_ITMP3);
4264
4265         /* save return value */
4266
4267         if (md->returntype.type != TYPE_VOID) {
4268                 if (IS_INT_LNG_TYPE(md->returntype.type))
4269                         M_LST(REG_RESULT, REG_SP, 0 * 8);
4270                 else
4271                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
4272         }
4273
4274         /* remove native stackframe info */
4275
4276         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4277         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
4278         M_CALL(REG_ITMP1);
4279
4280 #if !defined(NDEBUG)
4281         /* generate call trace */
4282
4283         if (opt_verbosecall) {
4284                 /* just restore the value we need, don't care about the other */
4285
4286                 if (md->returntype.type != TYPE_VOID) {
4287                         if (IS_INT_LNG_TYPE(md->returntype.type))
4288                                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
4289                         else
4290                                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4291                 }
4292
4293                 M_MOV_IMM(m, rd->argintregs[0]);
4294                 M_MOV(REG_RESULT, rd->argintregs[1]);
4295                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4296                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4297
4298                 M_MOV_IMM(builtin_displaymethodstop, REG_ITMP1);
4299                 M_CALL(REG_ITMP1);
4300         }
4301 #endif /* !defined(NDEBUG) */
4302
4303         /* check for exception */
4304
4305 #if defined(ENABLE_THREADS)
4306         M_MOV_IMM(builtin_get_exceptionptrptr, REG_ITMP3);
4307         M_CALL(REG_ITMP3);
4308 #else
4309         M_MOV_IMM(&_no_threads_exceptionptr, REG_RESULT);
4310 #endif
4311         M_ALD(REG_ITMP2, REG_RESULT, 0);
4312
4313         /* restore return value */
4314
4315         if (md->returntype.type != TYPE_VOID) {
4316                 if (IS_INT_LNG_TYPE(md->returntype.type))
4317                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
4318                 else
4319                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4320         }
4321
4322         /* test for exception */
4323
4324         M_TEST(REG_ITMP2);
4325         M_BNE(7 + 1);
4326
4327         /* remove stackframe */
4328
4329         M_AADD_IMM(stackframesize * 8, REG_SP);
4330         M_RET;
4331
4332
4333         /* handle exception */
4334
4335 #if defined(ENABLE_THREADS)
4336         M_LST(REG_ITMP2, REG_SP, 0 * 8);
4337         M_MOV_IMM(builtin_get_exceptionptrptr, REG_ITMP3);
4338         M_CALL(REG_ITMP3);
4339         M_AST_IMM32(0, REG_RESULT, 0);                 /* clear exception pointer */
4340         M_LLD(REG_ITMP1_XPTR, REG_SP, 0 * 8);
4341 #else
4342         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
4343         M_MOV_IMM(&_no_threads_exceptionptr, REG_ITMP3);
4344         M_AST_IMM32(0, REG_ITMP3, 0);                  /* clear exception pointer */
4345 #endif
4346
4347         /* remove stackframe */
4348
4349         M_AADD_IMM(stackframesize * 8, REG_SP);
4350
4351         M_LLD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
4352         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
4353
4354         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
4355         M_JMP(REG_ITMP3);
4356
4357
4358         /* process patcher calls **************************************************/
4359
4360         {
4361                 patchref *pref;
4362                 ptrint    mcode;
4363                 u1       *savedmcodeptr;
4364                 u1       *tmpmcodeptr;
4365 #if defined(ENABLE_THREADS)
4366                 s4        disp;
4367 #endif
4368
4369                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4370                         /* Get machine code which is patched back in later. A
4371                            `call rel32' is 5 bytes long (but read 8 bytes). */
4372
4373                         savedmcodeptr = cd->mcodebase + pref->branchpos;
4374                         mcode = *((ptrint *) savedmcodeptr);
4375
4376                         /* patch in `call rel32' to call the following code               */
4377
4378                         tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr          */
4379                         cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position */
4380
4381                         M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
4382
4383                         cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr   */
4384
4385                         /* move pointer to java_objectheader onto stack */
4386
4387 #if defined(ENABLE_THREADS)
4388                         /* create a virtual java_objectheader */
4389
4390                         (void) dseg_addaddress(cd, NULL);                         /* flcword    */
4391                         (void) dseg_addaddress(cd, lock_get_initial_lock_word()); /* monitorPtr */
4392                         disp = dseg_addaddress(cd, NULL);                         /* vftbl      */
4393
4394                         emit_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP3);
4395                         M_PUSH(REG_ITMP3);
4396 #else
4397                         M_PUSH_IMM(0);
4398 #endif
4399
4400                         /* move machine code bytes and classinfo pointer into registers */
4401
4402                         M_MOV_IMM(mcode, REG_ITMP3);
4403                         M_PUSH(REG_ITMP3);
4404                         M_MOV_IMM(pref->ref, REG_ITMP3);
4405                         M_PUSH(REG_ITMP3);
4406                         M_MOV_IMM(pref->disp, REG_ITMP3);
4407                         M_PUSH(REG_ITMP3);
4408
4409                         M_MOV_IMM(pref->patcher, REG_ITMP3);
4410                         M_PUSH(REG_ITMP3);
4411
4412                         M_MOV_IMM(asm_wrapper_patcher, REG_ITMP3);
4413                         M_JMP(REG_ITMP3);
4414                 }
4415         }
4416
4417         codegen_finish(jd);
4418
4419         return jd->code->entrypoint;
4420 }
4421
4422
4423 /*
4424  * These are local overrides for various environment variables in Emacs.
4425  * Please do not remove this and leave it at the end of the file, where
4426  * Emacs will automagically detect them.
4427  * ---------------------------------------------------------------------
4428  * Local variables:
4429  * mode: c
4430  * indent-tabs-mode: t
4431  * c-basic-offset: 4
4432  * tab-width: 4
4433  * End:
4434  * vim:noexpandtab:sw=4:ts=4:
4435  */