src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31             Edwin Steiner
  32
  33    $Id: codegen.c 4615 2006-03-15 16:36:43Z twisti $
  34
  35 */
  36
  37
  38 #include "config.h"
  39
  40 #include <stdio.h>
  41
  42 #include "vm/types.h"
  43
  44 #include "md.h"
  45 #include "md-abi.h"
  46
  47 #include "vm/jit/x86_64/arch.h"
  48 #include "vm/jit/x86_64/codegen.h"
  49 #include "vm/jit/x86_64/emitfuncs.h"
  50
  51 #include "native/native.h"
  52 #include "vm/builtin.h"
  53 #include "vm/exceptions.h"
  54 #include "vm/global.h"
  55 #include "vm/loader.h"
  56 #include "vm/options.h"
  57 #include "vm/statistics.h"
  58 #include "vm/stringlocal.h"
  59 #include "vm/vm.h"
  60 #include "vm/jit/asmpart.h"
  61 #include "vm/jit/codegen-common.h"
  62 #include "vm/jit/dseg.h"
  63 #include "vm/jit/jit.h"
  64 #include "vm/jit/methodheader.h"
  65 #include "vm/jit/parse.h"
  66 #include "vm/jit/patcher.h"
  67 #include "vm/jit/reg.h"
  68
  69 #if defined(ENABLE_LSRA)
  70 # include "vm/jit/allocator/lsra.h"
  71 #endif
  72
  73
  74
  75
  76 /* codegen *********************************************************************
  77
  78    Generates machine code.
  79
  80 *******************************************************************************/
  81
  82 bool codegen(methodinfo *m, codegendata *cd, registerdata *rd)
  83 {
  84         s4                  len, s1, s2, s3, d, disp;
  85         u2                  currentline;
  86         ptrint              a;
  87         s4                  parentargs_base;
  88         stackptr            src;
  89         varinfo            *var;
  90         basicblock         *bptr;
  91         instruction        *iptr;
  92         exceptiontable     *ex;
  93         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
  94         builtintable_entry *bte;
  95         methoddesc         *md;
  96
  97         /* prevent compiler warnings */
  98
  99         d = 0;
 100         lm = NULL;
 101         bte = NULL;
 102
 103         {
 104         s4 i, p, t, l;
 105         s4 savedregs_num;
 106
 107         savedregs_num = 0;
 108
 109         /* space to save used callee saved registers */
 110
 111         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 112         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 113
 114         parentargs_base = rd->memuse + savedregs_num;
 115
 116 #if defined(USE_THREADS)
 117         /* space to save argument of monitor_enter */
 118
 119         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 120                 parentargs_base++;
 121 #endif
 122
 123     /* Keep stack of non-leaf functions 16-byte aligned for calls into native */
 124         /* code e.g. libc or jni (alignment problems with movaps).                */
 125
 126         if (!m->isleafmethod || opt_verbosecall)
 127                 parentargs_base |= 0x1;
 128
 129         /* create method header */
 130
 131         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 132         (void) dseg_adds4(cd, parentargs_base * 8);             /* FrameSize      */
 133
 134 #if defined(USE_THREADS)
 135         /* IsSync contains the offset relative to the stack pointer for the
 136            argument of monitor_exit used in the exception handler. Since the
 137            offset could be zero and give a wrong meaning of the flag it is
 138            offset by one.
 139         */
 140
 141         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 142                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);        /* IsSync         */
 143         else
 144 #endif
 145                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 146
 147         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 148         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse);  /* IntSave        */
 149         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse);  /* FltSave        */
 150
 151         (void) dseg_addlinenumbertablesize(cd);
 152
 153         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 154
 155         /* create exception table */
 156
 157         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 158                 dseg_addtarget(cd, ex->start);
 159                 dseg_addtarget(cd, ex->end);
 160                 dseg_addtarget(cd, ex->handler);
 161                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 162         }
 163
 164         /* initialize mcode variables */
 165
 166         cd->mcodeptr = (u1 *) cd->mcodebase;
 167         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
 168
 169         /* initialize the last patcher pointer */
 170
 171         cd->lastmcodeptr = cd->mcodeptr;
 172
 173         /* generate method profiling code */
 174
 175         if (opt_prof) {
 176                 /* count frequency */
 177
 178                 M_MOV_IMM(m, REG_ITMP3);
 179                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(methodinfo, frequency));
 180
 181                 PROFILE_CYCLE_START;
 182         }
 183
 184         /* create stack frame (if necessary) */
 185
 186         if (parentargs_base)
 187                 M_ASUB_IMM(parentargs_base * 8, REG_SP);
 188
 189         /* save used callee saved registers */
 190
 191         p = parentargs_base;
 192         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 193                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 194         }
 195         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 196                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 197         }
 198
 199         /* take arguments out of register or stack frame */
 200
 201         md = m->parseddesc;
 202
 203         for (p = 0, l = 0; p < md->paramcount; p++) {
 204                 t = md->paramtypes[p].type;
 205                 var = &(rd->locals[l][t]);
 206                 l++;
 207                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 208                         l++;
 209                 if (var->type < 0)
 210                         continue;
 211                 s1 = md->params[p].regoff;
 212                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 213                         s2 = rd->argintregs[s1];
 214                         if (!md->params[p].inmemory) {           /* register arguments    */
 215                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 216                                         M_INTMOVE(s2, var->regoff);
 217
 218                                 } else {                             /* reg arg -> spilled    */
 219                                     M_LST(s2, REG_SP, var->regoff * 8);
 220                                 }
 221
 222                         } else {                                 /* stack arguments       */
 223                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 224                                         /* + 8 for return address */
 225                                         M_LLD(var->regoff, REG_SP, (parentargs_base + s1) * 8 + 8);
 226
 227                                 } else {                             /* stack arg -> spilled  */
 228                                         var->regoff = parentargs_base + s1 + 1;
 229                                 }
 230                         }
 231
 232                 } else {                                     /* floating args         */
 233                         if (!md->params[p].inmemory) {           /* register arguments    */
 234                                 s2 = rd->argfltregs[s1];
 235                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 236                                         M_FLTMOVE(s2, var->regoff);
 237
 238                                 } else {                                         /* reg arg -> spilled    */
 239                                         M_DST(s2, REG_SP, var->regoff * 8);
 240                                 }
 241
 242                         } else {                                 /* stack arguments       */
 243                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 244                                         M_DLD(var->regoff, REG_SP, (parentargs_base + s1) * 8 + 8);
 245
 246                                 } else {
 247                                         var->regoff = parentargs_base + s1 + 1;
 248                                 }
 249                         }
 250                 }
 251         }  /* end for */
 252
 253         /* save monitorenter argument */
 254
 255 #if defined(USE_THREADS)
 256         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 257                 /* stack offset for monitor argument */
 258
 259                 s1 = rd->memuse;
 260
 261                 if (opt_verbosecall) {
 262                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 263
 264                         for (p = 0; p < INT_ARG_CNT; p++)
 265                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 266
 267                         for (p = 0; p < FLT_ARG_CNT; p++)
 268                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 269
 270                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 271                 }
 272
 273                 /* decide which monitor enter function to call */
 274
 275                 if (m->flags & ACC_STATIC) {
 276                         M_MOV_IMM(m->class, REG_ITMP1);
 277                         M_AST(REG_ITMP1, REG_SP, s1 * 8);
 278                         M_INTMOVE(REG_ITMP1, rd->argintregs[0]);
 279                         M_MOV_IMM(BUILTIN_staticmonitorenter, REG_ITMP1);
 280                         M_CALL(REG_ITMP1);
 281
 282                 } else {
 283                         M_TEST(rd->argintregs[0]);
 284                         M_BEQ(0);
 285                         codegen_add_nullpointerexception_ref(cd, cd->mcodeptr);
 286                         M_AST(rd->argintregs[0], REG_SP, s1 * 8);
 287                         M_MOV_IMM(BUILTIN_monitorenter, REG_ITMP1);
 288                         M_CALL(REG_ITMP1);
 289                 }
 290
 291                 if (opt_verbosecall) {
 292                         for (p = 0; p < INT_ARG_CNT; p++)
 293                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 294
 295                         for (p = 0; p < FLT_ARG_CNT; p++)
 296                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 297
 298                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 299                 }
 300         }
 301 #endif
 302
 303         /* Copy argument registers to stack and call trace function with
 304            pointer to arguments on stack. */
 305
 306         if (opt_verbosecall) {
 307                 M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 308
 309                 /* save integer argument registers */
 310
 311                 for (p = 0; p < INT_ARG_CNT; p++)
 312                         M_LST(rd->argintregs[p], REG_SP, (1 + p) * 8);
 313
 314                 /* save float argument registers */
 315
 316                 for (p = 0; p < FLT_ARG_CNT; p++)
 317                         M_DST(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 318
 319                 /* save temporary registers for leaf methods */
 320
 321                 if (m->isleafmethod) {
 322                         for (p = 0; p < INT_TMP_CNT; p++)
 323                                 M_LST(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 324
 325                         for (p = 0; p < FLT_TMP_CNT; p++)
 326                                 M_DST(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 327                 }
 328
 329                 /* show integer hex code for float arguments */
 330
 331                 for (p = 0, l = 0; p < md->paramcount && p < INT_ARG_CNT; p++) {
 332                         /* If the paramtype is a float, we have to right shift all
 333                            following integer registers. */
 334
 335                         if (IS_FLT_DBL_TYPE(md->paramtypes[p].type)) {
 336                                 for (s1 = INT_ARG_CNT - 2; s1 >= p; s1--) {
 337                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
 338                                 }
 339
 340                                 x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 341                                 l++;
 342                         }
 343                 }
 344
 345                 M_MOV_IMM(m, REG_ITMP2);
 346                 M_AST(REG_ITMP2, REG_SP, 0 * 8);
 347                 M_MOV_IMM(builtin_trace_args, REG_ITMP1);
 348                 M_CALL(REG_ITMP1);
 349
 350                 /* restore integer argument registers */
 351
 352                 for (p = 0; p < INT_ARG_CNT; p++)
 353                         M_LLD(rd->argintregs[p], REG_SP, (1 + p) * 8);
 354
 355                 /* restore float argument registers */
 356
 357                 for (p = 0; p < FLT_ARG_CNT; p++)
 358                         M_DLD(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 359
 360                 /* restore temporary registers for leaf methods */
 361
 362                 if (m->isleafmethod) {
 363                         for (p = 0; p < INT_TMP_CNT; p++)
 364                                 M_LLD(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 365
 366                         for (p = 0; p < FLT_TMP_CNT; p++)
 367                                 M_DLD(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 368                 }
 369
 370                 M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 371         }
 372
 373         }
 374
 375         /* end of header generation */
 376
 377         /* walk through all basic blocks */
 378
 379         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 380
 381                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 382
 383                 if (bptr->flags >= BBREACHED) {
 384
 385                         /* branch resolving */
 386
 387                         branchref *bref;
 388                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 389                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 390                                                                   bref->branchpos,
 391                                                                   bptr->mpc);
 392                         }
 393
 394                 /* copy interface registers to their destination */
 395
 396                 src = bptr->instack;
 397                 len = bptr->indepth;
 398                 MCODECHECK(512);
 399
 400                 /* generate basicblock profiling code */
 401
 402                 if (opt_prof_bb) {
 403                         /* count frequency */
 404
 405                         M_MOV_IMM(m->bbfrequency, REG_ITMP2);
 406                         M_IINC_MEMBASE(REG_ITMP2, bptr->debug_nr * 4);
 407
 408                         /* if this is an exception handler, start profiling again */
 409
 410                         if (bptr->type == BBTYPE_EXH)
 411                                 PROFILE_CYCLE_START;
 412                 }
 413
 414 #if defined(ENABLE_LSRA)
 415                 if (opt_lsra) {
 416                         while (src != NULL) {
 417                                 len--;
 418                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 419                                         if (bptr->type == BBTYPE_SBR) {
 420                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 421                                                 if (!(src->flags & INMEMORY))
 422                                                         d= src->regoff;
 423                                                 else
 424                                                         d=REG_ITMP1;
 425                                                 x86_64_pop_reg(cd, d);
 426                                                 store_reg_to_var_int(src, d);
 427
 428                                         } else if (bptr->type == BBTYPE_EXH) {
 429                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 430                                                 if (!(src->flags & INMEMORY))
 431                                                         d= src->regoff;
 432                                                 else
 433                                                         d=REG_ITMP1;
 434                                                 M_INTMOVE(REG_ITMP1, d);
 435                                                 store_reg_to_var_int(src, d);
 436                                         }
 437                                 }
 438                                 src = src->prev;
 439                         }
 440
 441                 } else {
 442 #endif
 443
 444                 while (src != NULL) {
 445                         len--;
 446                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 447                                 if (bptr->type == BBTYPE_SBR) {
 448                                         d = reg_of_var(rd, src, REG_ITMP1);
 449                                         M_POP(d);
 450                                         store_reg_to_var_int(src, d);
 451
 452                                 } else if (bptr->type == BBTYPE_EXH) {
 453                                         d = reg_of_var(rd, src, REG_ITMP1);
 454                                         M_INTMOVE(REG_ITMP1, d);
 455                                         store_reg_to_var_int(src, d);
 456                                 }
 457
 458                         } else {
 459                                 d = reg_of_var(rd, src, REG_ITMP1);
 460                                 if ((src->varkind != STACKVAR)) {
 461                                         s2 = src->type;
 462                                         if (IS_FLT_DBL_TYPE(s2)) {
 463                                                 s1 = rd->interfaces[len][s2].regoff;
 464
 465                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 466                                                         M_FLTMOVE(s1, d);
 467                                                 else
 468                                                         M_DLD(d, REG_SP, s1 * 8);
 469
 470                                                 store_reg_to_var_flt(src, d);
 471
 472                                         } else {
 473                                                 s1 = rd->interfaces[len][s2].regoff;
 474
 475                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 476                                                         M_INTMOVE(s1, d);
 477                                                 else
 478                                                         M_LLD(d, REG_SP, s1 * 8);
 479
 480                                                 store_reg_to_var_int(src, d);
 481                                         }
 482                                 }
 483                         }
 484                         src = src->prev;
 485                 }
 486 #if defined(ENABLE_LSRA)
 487                 }
 488 #endif
 489                 /* walk through all instructions */
 490
 491                 src = bptr->instack;
 492                 len = bptr->icount;
 493                 currentline = 0;
 494
 495                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 496                         if (iptr->line != currentline) {
 497                                 dseg_addlinenumber(cd, iptr->line, cd->mcodeptr);
 498                                 currentline = iptr->line;
 499                         }
 500
 501                         MCODECHECK(1024);                         /* 1KB should be enough */
 502
 503                         switch (iptr->opc) {
 504                         case ICMD_INLINE_START: /* internal ICMDs                         */
 505                         case ICMD_INLINE_END:
 506                                 break;
 507
 508                         case ICMD_NOP:    /* ...  ==> ...                                 */
 509                                 break;
 510
 511                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 512
 513                                 if (src->flags & INMEMORY)
 514                                         M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
 515                                 else
 516                                         M_TEST(src->regoff);
 517                                 M_BEQ(0);
 518                                 codegen_add_nullpointerexception_ref(cd, cd->mcodeptr);
 519                                 break;
 520
 521                 /* constant operations ************************************************/
 522
 523                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 524                                       /* op1 = 0, val.i = constant                    */
 525
 526                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 527                         if (iptr->val.i == 0)
 528                                 M_CLR(d);
 529                         else
 530                                 M_IMOV_IMM(iptr->val.i, d);
 531                         store_reg_to_var_int(iptr->dst, d);
 532                         break;
 533
 534                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 535                                       /* op1 = 0, val.l = constant                    */
 536
 537                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 538                         if (iptr->val.l == 0)
 539                                 M_CLR(d);
 540                         else
 541                                 M_MOV_IMM(iptr->val.l, d);
 542                         store_reg_to_var_int(iptr->dst, d);
 543                         break;
 544
 545                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 546                                       /* op1 = 0, val.f = constant                    */
 547
 548                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 549                         disp = dseg_addfloat(cd, iptr->val.f);
 550                         x86_64_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + disp, d);
 551                         store_reg_to_var_flt(iptr->dst, d);
 552                         break;
 553
 554                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 555                                       /* op1 = 0, val.d = constant                    */
 556
 557                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 558                         disp = dseg_adddouble(cd, iptr->val.d);
 559                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, d);
 560                         store_reg_to_var_flt(iptr->dst, d);
 561                         break;
 562
 563                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 564                                       /* op1 = 0, val.a = constant                    */
 565
 566                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 567
 568                         if ((iptr->target != NULL) && (iptr->val.a == NULL)) {
 569 /*                              PROFILE_CYCLE_STOP; */
 570
 571                                 codegen_addpatchref(cd, cd->mcodeptr,
 572                                                                         PATCHER_aconst,
 573                                                                         (unresolved_class *) iptr->target, 0);
 574
 575                                 if (opt_showdisassemble) {
 576                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
 577                                 }
 578
 579 /*                              PROFILE_CYCLE_START; */
 580
 581                                 M_MOV_IMM(iptr->val.a, d);
 582
 583                         } else {
 584                                 if (iptr->val.a == 0)
 585                                         M_CLR(d);
 586                                 else
 587                                         M_MOV_IMM(iptr->val.a, d);
 588                         }
 589                         store_reg_to_var_int(iptr->dst, d);
 590                         break;
 591
 592
 593                 /* load/store operations **********************************************/
 594
 595                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 596                                       /* op1 = local variable                         */
 597
 598                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 599                         if ((iptr->dst->varkind == LOCALVAR) &&
 600                             (iptr->dst->varnum == iptr->op1)) {
 601                                 break;
 602                         }
 603                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 604                         if (var->flags & INMEMORY) {
 605                                 x86_64_movl_membase_reg(cd, REG_SP, var->regoff * 8, d);
 606                                 store_reg_to_var_int(iptr->dst, d);
 607
 608                         } else {
 609                                 if (iptr->dst->flags & INMEMORY) {
 610                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 611
 612                                 } else {
 613                                         M_INTMOVE(var->regoff, d);
 614                                 }
 615                         }
 616                         break;
 617
 618                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 619                 case ICMD_ALOAD:      /* op1 = local variable                         */
 620
 621                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 622                         if ((iptr->dst->varkind == LOCALVAR) &&
 623                             (iptr->dst->varnum == iptr->op1)) {
 624                                 break;
 625                         }
 626                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 627                         if (var->flags & INMEMORY) {
 628                                 x86_64_mov_membase_reg(cd, REG_SP, var->regoff * 8, d);
 629                                 store_reg_to_var_int(iptr->dst, d);
 630
 631                         } else {
 632                                 if (iptr->dst->flags & INMEMORY) {
 633                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 634
 635                                 } else {
 636                                         M_INTMOVE(var->regoff, d);
 637                                 }
 638                         }
 639                         break;
 640
 641                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 642                 case ICMD_DLOAD:      /* op1 = local variable                         */
 643
 644                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 645                         if ((iptr->dst->varkind == LOCALVAR) &&
 646                             (iptr->dst->varnum == iptr->op1)) {
 647                                 break;
 648                         }
 649                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 650                         if (var->flags & INMEMORY) {
 651                                 x86_64_movq_membase_reg(cd, REG_SP, var->regoff * 8, d);
 652                                 store_reg_to_var_flt(iptr->dst, d);
 653
 654                         } else {
 655                                 if (iptr->dst->flags & INMEMORY) {
 656                                         x86_64_movq_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 657
 658                                 } else {
 659                                         M_FLTMOVE(var->regoff, d);
 660                                 }
 661                         }
 662                         break;
 663
 664                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 665                 case ICMD_LSTORE:     /* op1 = local variable                         */
 666                 case ICMD_ASTORE:
 667
 668                         if ((src->varkind == LOCALVAR) &&
 669                             (src->varnum == iptr->op1)) {
 670                                 break;
 671                         }
 672                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 673                         if (var->flags & INMEMORY) {
 674                                 var_to_reg_int(s1, src, REG_ITMP1);
 675                                 x86_64_mov_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 676
 677                         } else {
 678                                 var_to_reg_int(s1, src, var->regoff);
 679                                 M_INTMOVE(s1, var->regoff);
 680                         }
 681                         break;
 682
 683                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 684                 case ICMD_DSTORE:     /* op1 = local variable                         */
 685
 686                         if ((src->varkind == LOCALVAR) &&
 687                             (src->varnum == iptr->op1)) {
 688                                 break;
 689                         }
 690                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 691                         if (var->flags & INMEMORY) {
 692                                 var_to_reg_flt(s1, src, REG_FTMP1);
 693                                 x86_64_movq_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 694
 695                         } else {
 696                                 var_to_reg_flt(s1, src, var->regoff);
 697                                 M_FLTMOVE(s1, var->regoff);
 698                         }
 699                         break;
 700
 701
 702                 /* pop/dup/swap operations ********************************************/
 703
 704                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 705
 706                 case ICMD_POP:        /* ..., value  ==> ...                          */
 707                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 708                         break;
 709
 710                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 711                         M_COPY(src, iptr->dst);
 712                         break;
 713
 714                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 715
 716                         M_COPY(src,       iptr->dst);
 717                         M_COPY(src->prev, iptr->dst->prev);
 718                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 719                         break;
 720
 721                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 722
 723                         M_COPY(src,             iptr->dst);
 724                         M_COPY(src->prev,       iptr->dst->prev);
 725                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 726                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 727                         break;
 728
 729                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 730
 731                         M_COPY(src,       iptr->dst);
 732                         M_COPY(src->prev, iptr->dst->prev);
 733                         break;
 734
 735                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 736
 737                         M_COPY(src,             iptr->dst);
 738                         M_COPY(src->prev,       iptr->dst->prev);
 739                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 740                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 741                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 742                         break;
 743
 744                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 745
 746                         M_COPY(src,                   iptr->dst);
 747                         M_COPY(src->prev,             iptr->dst->prev);
 748                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 749                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 750                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 751                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 752                         break;
 753
 754                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 755
 756                         M_COPY(src,       iptr->dst->prev);
 757                         M_COPY(src->prev, iptr->dst);
 758                         break;
 759
 760
 761                 /* integer operations *************************************************/
 762
 763                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 764
 765                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 766                         if (iptr->dst->flags & INMEMORY) {
 767                                 if (src->flags & INMEMORY) {
 768                                         if (src->regoff == iptr->dst->regoff) {
 769                                                 x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 770
 771                                         } else {
 772                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 773                                                 x86_64_negl_reg(cd, REG_ITMP1);
 774                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 775                                         }
 776
 777                                 } else {
 778                                         x86_64_movl_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 779                                         x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 780                                 }
 781
 782                         } else {
 783                                 if (src->flags & INMEMORY) {
 784                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 785                                         x86_64_negl_reg(cd, d);
 786
 787                                 } else {
 788                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 789                                         x86_64_negl_reg(cd, iptr->dst->regoff);
 790                                 }
 791                         }
 792                         break;
 793
 794                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 795
 796                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 797                         if (iptr->dst->flags & INMEMORY) {
 798                                 if (src->flags & INMEMORY) {
 799                                         if (src->regoff == iptr->dst->regoff) {
 800                                                 x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 801
 802                                         } else {
 803                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 804                                                 x86_64_neg_reg(cd, REG_ITMP1);
 805                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 806                                         }
 807
 808                                 } else {
 809                                         x86_64_mov_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 810                                         x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 811                                 }
 812
 813                         } else {
 814                                 if (src->flags & INMEMORY) {
 815                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 816                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 817
 818                                 } else {
 819                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 820                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 821                                 }
 822                         }
 823                         break;
 824
 825                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 826
 827                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 828                         if (src->flags & INMEMORY) {
 829                                 x86_64_movslq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 830
 831                         } else {
 832                                 x86_64_movslq_reg_reg(cd, src->regoff, d);
 833                         }
 834                         store_reg_to_var_int(iptr->dst, d);
 835                         break;
 836
 837                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 838
 839                         var_to_reg_int(s1, src, REG_ITMP1);
 840                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 841                         M_INTMOVE(s1, d);
 842                         store_reg_to_var_int(iptr->dst, d);
 843                         break;
 844
 845                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 846
 847                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 848                         if (src->flags & INMEMORY) {
 849                                 x86_64_movsbq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 850
 851                         } else {
 852                                 x86_64_movsbq_reg_reg(cd, src->regoff, d);
 853                         }
 854                         store_reg_to_var_int(iptr->dst, d);
 855                         break;
 856
 857                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 858
 859                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 860                         if (src->flags & INMEMORY) {
 861                                 x86_64_movzwq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 862
 863                         } else {
 864                                 x86_64_movzwq_reg_reg(cd, src->regoff, d);
 865                         }
 866                         store_reg_to_var_int(iptr->dst, d);
 867                         break;
 868
 869                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 870
 871                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 872                         if (src->flags & INMEMORY) {
 873                                 x86_64_movswq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 874
 875                         } else {
 876                                 x86_64_movswq_reg_reg(cd, src->regoff, d);
 877                         }
 878                         store_reg_to_var_int(iptr->dst, d);
 879                         break;
 880
 881
 882                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 883
 884                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 885                         x86_64_emit_ialu(cd, X86_64_ADD, src, iptr);
 886                         break;
 887
 888                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 889                                       /* val.i = constant                             */
 890
 891                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 892                         x86_64_emit_ialuconst(cd, X86_64_ADD, src, iptr);
 893                         break;
 894
 895                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 896
 897                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 898                         x86_64_emit_lalu(cd, X86_64_ADD, src, iptr);
 899                         break;
 900
 901                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 902                                       /* val.l = constant                             */
 903
 904                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 905                         x86_64_emit_laluconst(cd, X86_64_ADD, src, iptr);
 906                         break;
 907
 908                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 909
 910                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 911                         if (iptr->dst->flags & INMEMORY) {
 912                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 913                                         if (src->prev->regoff == iptr->dst->regoff) {
 914                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 915                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 916
 917                                         } else {
 918                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 919                                                 x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 920                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 921                                         }
 922
 923                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 924                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 925                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 926                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 927
 928                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 929                                         if (src->prev->regoff == iptr->dst->regoff) {
 930                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 931
 932                                         } else {
 933                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 934                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 935                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 936                                         }
 937
 938                                 } else {
 939                                         x86_64_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 940                                         x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 941                                 }
 942
 943                         } else {
 944                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 945                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 946                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 947
 948                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 949                                         M_INTMOVE(src->prev->regoff, d);
 950                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 951
 952                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 953                                         /* workaround for reg alloc */
 954                                         if (src->regoff == iptr->dst->regoff) {
 955                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 956                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 957                                                 M_INTMOVE(REG_ITMP1, d);
 958
 959                                         } else {
 960                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 961                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 962                                         }
 963
 964                                 } else {
 965                                         /* workaround for reg alloc */
 966                                         if (src->regoff == iptr->dst->regoff) {
 967                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
 968                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 969                                                 M_INTMOVE(REG_ITMP1, d);
 970
 971                                         } else {
 972                                                 M_INTMOVE(src->prev->regoff, d);
 973                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 974                                         }
 975                                 }
 976                         }
 977                         break;
 978
 979                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 980                                       /* val.i = constant                             */
 981
 982                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 983                         x86_64_emit_ialuconst(cd, X86_64_SUB, src, iptr);
 984                         break;
 985
 986                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 987
 988                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 989                         if (iptr->dst->flags & INMEMORY) {
 990                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 991                                         if (src->prev->regoff == iptr->dst->regoff) {
 992                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 993                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 994
 995                                         } else {
 996                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 997                                                 x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 998                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 999                                         }
1000
1001                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1002                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1003                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1004                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1005
1006                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1007                                         if (src->prev->regoff == iptr->dst->regoff) {
1008                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1009
1010                                         } else {
1011                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1012                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1013                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1014                                         }
1015
1016                                 } else {
1017                                         x86_64_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
1018                                         x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1019                                 }
1020
1021                         } else {
1022                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1023                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1024                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1025
1026                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1027                                         M_INTMOVE(src->prev->regoff, d);
1028                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1029
1030                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1031                                         /* workaround for reg alloc */
1032                                         if (src->regoff == iptr->dst->regoff) {
1033                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1034                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1035                                                 M_INTMOVE(REG_ITMP1, d);
1036
1037                                         } else {
1038                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1039                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1040                                         }
1041
1042                                 } else {
1043                                         /* workaround for reg alloc */
1044                                         if (src->regoff == iptr->dst->regoff) {
1045                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1046                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1047                                                 M_INTMOVE(REG_ITMP1, d);
1048
1049                                         } else {
1050                                                 M_INTMOVE(src->prev->regoff, d);
1051                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1052                                         }
1053                                 }
1054                         }
1055                         break;
1056
1057                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1058                                       /* val.l = constant                             */
1059
1060                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1061                         x86_64_emit_laluconst(cd, X86_64_SUB, src, iptr);
1062                         break;
1063
1064                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1065
1066                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1067                         if (iptr->dst->flags & INMEMORY) {
1068                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1069                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1070                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1071                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1072
1073                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1074                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1075                                         x86_64_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1076                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1077
1078                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1079                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1080                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1081                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1082
1083                                 } else {
1084                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1085                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1086                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1087                                 }
1088
1089                         } else {
1090                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1091                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1092                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1093
1094                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1095                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1096                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1097
1098                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1099                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1100                                         x86_64_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1101
1102                                 } else {
1103                                         if (src->regoff == iptr->dst->regoff) {
1104                                                 x86_64_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1105
1106                                         } else {
1107                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1108                                                 x86_64_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1109                                         }
1110                                 }
1111                         }
1112                         break;
1113
1114                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1115                                       /* val.i = constant                             */
1116
1117                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1118                         if (iptr->dst->flags & INMEMORY) {
1119                                 if (src->flags & INMEMORY) {
1120                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1121                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1122
1123                                 } else {
1124                                         x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1125                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1126                                 }
1127
1128                         } else {
1129                                 if (src->flags & INMEMORY) {
1130                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1131
1132                                 } else {
1133                                         if (iptr->val.i == 2) {
1134                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1135                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1136
1137                                         } else {
1138                                                 x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1139                                         }
1140                                 }
1141                         }
1142                         break;
1143
1144                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1145
1146                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1147                         if (iptr->dst->flags & INMEMORY) {
1148                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1149                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1150                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1151                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1152
1153                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1154                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1155                                         x86_64_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1156                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1157
1158                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1159                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1160                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1161                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1162
1163                                 } else {
1164                                         x86_64_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1165                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1166                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1167                                 }
1168
1169                         } else {
1170                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1171                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1172                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1173
1174                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1175                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1176                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1177
1178                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1179                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1180                                         x86_64_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1181
1182                                 } else {
1183                                         if (src->regoff == iptr->dst->regoff) {
1184                                                 x86_64_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1185
1186                                         } else {
1187                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1188                                                 x86_64_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1189                                         }
1190                                 }
1191                         }
1192                         break;
1193
1194                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1195                                       /* val.l = constant                             */
1196
1197                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1198                         if (iptr->dst->flags & INMEMORY) {
1199                                 if (src->flags & INMEMORY) {
1200                                         if (IS_IMM32(iptr->val.l)) {
1201                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1202
1203                                         } else {
1204                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1205                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1206                                         }
1207                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1208
1209                                 } else {
1210                                         if (IS_IMM32(iptr->val.l)) {
1211                                                 x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1212
1213                                         } else {
1214                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1215                                                 x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1216                                         }
1217                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1218                                 }
1219
1220                         } else {
1221                                 if (src->flags & INMEMORY) {
1222                                         if (IS_IMM32(iptr->val.l)) {
1223                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1224
1225                                         } else {
1226                                                 x86_64_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1227                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1228                                         }
1229
1230                                 } else {
1231                                         /* should match in many cases */
1232                                         if (iptr->val.l == 2) {
1233                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1234                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1235
1236                                         } else {
1237                                                 if (IS_IMM32(iptr->val.l)) {
1238                                                         x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1239
1240                                                 } else {
1241                                                         x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1242                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1243                                                         x86_64_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1244                                                 }
1245                                         }
1246                                 }
1247                         }
1248                         break;
1249
1250                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1251
1252                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1253                 if (src->prev->flags & INMEMORY) {
1254                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1255
1256                         } else {
1257                                 M_INTMOVE(src->prev->regoff, RAX);
1258                         }
1259
1260                         if (src->flags & INMEMORY) {
1261                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1262
1263                         } else {
1264                                 M_INTMOVE(src->regoff, REG_ITMP3);
1265                         }
1266                         gen_div_check(src);
1267
1268                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1269                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1270                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1271                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                /* 6 bytes */
1272
1273                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1274                         x86_64_cltd(cd);
1275                         x86_64_idivl_reg(cd, REG_ITMP3);
1276
1277                         if (iptr->dst->flags & INMEMORY) {
1278                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1279                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1280
1281                         } else {
1282                                 M_INTMOVE(RAX, iptr->dst->regoff);
1283
1284                                 if (iptr->dst->regoff != RDX) {
1285                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1286                                 }
1287                         }
1288                         break;
1289
1290                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1291                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1292                         if (src->prev->flags & INMEMORY) {
1293                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1294
1295                         } else {
1296                                 M_INTMOVE(src->prev->regoff, RAX);
1297                         }
1298
1299                         if (src->flags & INMEMORY) {
1300                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1301
1302                         } else {
1303                                 M_INTMOVE(src->regoff, REG_ITMP3);
1304                         }
1305                         gen_div_check(src);
1306
1307                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1308
1309                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1310                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1311
1312
1313                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1314                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1315                         x86_64_jcc(cd, X86_64_CC_E, 1 + 3);                    /* 6 bytes */
1316
1317                         x86_64_cltd(cd);
1318                         x86_64_idivl_reg(cd, REG_ITMP3);
1319
1320                         if (iptr->dst->flags & INMEMORY) {
1321                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1322                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1323
1324                         } else {
1325                                 M_INTMOVE(RDX, iptr->dst->regoff);
1326
1327                                 if (iptr->dst->regoff != RDX) {
1328                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1329                                 }
1330                         }
1331                         break;
1332
1333                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1334                                       /* val.i = constant                             */
1335
1336                         var_to_reg_int(s1, src, REG_ITMP1);
1337                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1338                         M_INTMOVE(s1, REG_ITMP1);
1339                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1340                         x86_64_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1341                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1342                         x86_64_shiftl_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1343                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1344                         store_reg_to_var_int(iptr->dst, d);
1345                         break;
1346
1347                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1348                                       /* val.i = constant                             */
1349
1350                         var_to_reg_int(s1, src, REG_ITMP1);
1351                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1352                         M_INTMOVE(s1, REG_ITMP1);
1353                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1354                         x86_64_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1355                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1356                         x86_64_alul_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1357                         x86_64_alul_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1358                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1359                         store_reg_to_var_int(iptr->dst, d);
1360                         break;
1361
1362
1363                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1364
1365                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1366
1367                 if (src->prev->flags & INMEMORY) {
1368                                 M_LLD(RAX, REG_SP, src->prev->regoff * 8);
1369
1370                         } else {
1371                                 M_INTMOVE(src->prev->regoff, RAX);
1372                         }
1373
1374                         if (src->flags & INMEMORY) {
1375                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1376
1377                         } else {
1378                                 M_INTMOVE(src->regoff, REG_ITMP3);
1379                         }
1380                         gen_div_check(src);
1381
1382                         /* check as described in jvm spec */
1383                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1384                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, RAX);
1385                         M_BNE(4 + 6);
1386                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1387                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
1388
1389                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1390                         x86_64_cqto(cd);
1391                         x86_64_idiv_reg(cd, REG_ITMP3);
1392
1393                         if (iptr->dst->flags & INMEMORY) {
1394                                 M_LST(RAX, REG_SP, iptr->dst->regoff * 8);
1395                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1396
1397                         } else {
1398                                 M_INTMOVE(RAX, iptr->dst->regoff);
1399
1400                                 if (iptr->dst->regoff != RDX) {
1401                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1402                                 }
1403                         }
1404                         break;
1405
1406                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1407
1408                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1409                         if (src->prev->flags & INMEMORY) {
1410                                 M_LLD(REG_ITMP1, REG_SP, src->prev->regoff * 8);
1411
1412                         } else {
1413                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1414                         }
1415
1416                         if (src->flags & INMEMORY) {
1417                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1418
1419                         } else {
1420                                 M_INTMOVE(src->regoff, REG_ITMP3);
1421                         }
1422                         gen_div_check(src);
1423
1424                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1425
1426                         /* check as described in jvm spec */
1427                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1428                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP1);
1429                         M_BNE(3 + 4 + 6);
1430
1431 #if 0
1432                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1433 #endif
1434                         M_XOR(RDX, RDX);                                       /* 3 bytes */
1435                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1436                         M_BEQ(2 + 3);                                          /* 6 bytes */
1437
1438                         x86_64_cqto(cd);
1439                         x86_64_idiv_reg(cd, REG_ITMP3);
1440
1441                         if (iptr->dst->flags & INMEMORY) {
1442                                 M_LST(RDX, REG_SP, iptr->dst->regoff * 8);
1443                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1444
1445                         } else {
1446                                 M_INTMOVE(RDX, iptr->dst->regoff);
1447
1448                                 if (iptr->dst->regoff != RDX) {
1449                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1450                                 }
1451                         }
1452                         break;
1453
1454                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1455                                       /* val.i = constant                             */
1456
1457                         var_to_reg_int(s1, src, REG_ITMP1);
1458                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1459                         M_INTMOVE(s1, REG_ITMP1);
1460                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1461                         x86_64_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1462                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1463                         x86_64_shift_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1464                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1465                         store_reg_to_var_int(iptr->dst, d);
1466                         break;
1467
1468                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1469                                       /* val.l = constant                             */
1470
1471                         var_to_reg_int(s1, src, REG_ITMP1);
1472                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1473                         M_INTMOVE(s1, REG_ITMP1);
1474                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1475                         x86_64_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1476                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1477                         x86_64_alu_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1478                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1479                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1480                         store_reg_to_var_int(iptr->dst, d);
1481                         break;
1482
1483                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1484
1485                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1486                         x86_64_emit_ishift(cd, X86_64_SHL, src, iptr);
1487                         break;
1488
1489                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1490                                       /* val.i = constant                             */
1491
1492                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1493                         x86_64_emit_ishiftconst(cd, X86_64_SHL, src, iptr);
1494                         break;
1495
1496                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1497
1498                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1499                         x86_64_emit_ishift(cd, X86_64_SAR, src, iptr);
1500                         break;
1501
1502                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1503                                       /* val.i = constant                             */
1504
1505                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1506                         x86_64_emit_ishiftconst(cd, X86_64_SAR, src, iptr);
1507                         break;
1508
1509                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1510
1511                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1512                         x86_64_emit_ishift(cd, X86_64_SHR, src, iptr);
1513                         break;
1514
1515                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1516                                       /* val.i = constant                             */
1517
1518                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1519                         x86_64_emit_ishiftconst(cd, X86_64_SHR, src, iptr);
1520                         break;
1521
1522                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1523
1524                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1525                         x86_64_emit_lshift(cd, X86_64_SHL, src, iptr);
1526                         break;
1527
1528         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1529                                           /* val.i = constant                             */
1530
1531                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1532                         x86_64_emit_lshiftconst(cd, X86_64_SHL, src, iptr);
1533                         break;
1534
1535                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1536
1537                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1538                         x86_64_emit_lshift(cd, X86_64_SAR, src, iptr);
1539                         break;
1540
1541                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1542                                       /* val.i = constant                             */
1543
1544                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1545                         x86_64_emit_lshiftconst(cd, X86_64_SAR, src, iptr);
1546                         break;
1547
1548                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1549
1550                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1551                         x86_64_emit_lshift(cd, X86_64_SHR, src, iptr);
1552                         break;
1553
1554                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1555                                       /* val.l = constant                             */
1556
1557                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1558                         x86_64_emit_lshiftconst(cd, X86_64_SHR, src, iptr);
1559                         break;
1560
1561                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1562
1563                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1564                         x86_64_emit_ialu(cd, X86_64_AND, src, iptr);
1565                         break;
1566
1567                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1568                                       /* val.i = constant                             */
1569
1570                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1571                         x86_64_emit_ialuconst(cd, X86_64_AND, src, iptr);
1572                         break;
1573
1574                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1575
1576                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1577                         x86_64_emit_lalu(cd, X86_64_AND, src, iptr);
1578                         break;
1579
1580                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1581                                       /* val.l = constant                             */
1582
1583                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1584                         x86_64_emit_laluconst(cd, X86_64_AND, src, iptr);
1585                         break;
1586
1587                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1588
1589                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1590                         x86_64_emit_ialu(cd, X86_64_OR, src, iptr);
1591                         break;
1592
1593                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1594                                       /* val.i = constant                             */
1595
1596                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1597                         x86_64_emit_ialuconst(cd, X86_64_OR, src, iptr);
1598                         break;
1599
1600                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1601
1602                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1603                         x86_64_emit_lalu(cd, X86_64_OR, src, iptr);
1604                         break;
1605
1606                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1607                                       /* val.l = constant                             */
1608
1609                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1610                         x86_64_emit_laluconst(cd, X86_64_OR, src, iptr);
1611                         break;
1612
1613                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1614
1615                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1616                         x86_64_emit_ialu(cd, X86_64_XOR, src, iptr);
1617                         break;
1618
1619                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1620                                       /* val.i = constant                             */
1621
1622                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1623                         x86_64_emit_ialuconst(cd, X86_64_XOR, src, iptr);
1624                         break;
1625
1626                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1627
1628                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1629                         x86_64_emit_lalu(cd, X86_64_XOR, src, iptr);
1630                         break;
1631
1632                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1633                                       /* val.l = constant                             */
1634
1635                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1636                         x86_64_emit_laluconst(cd, X86_64_XOR, src, iptr);
1637                         break;
1638
1639
1640                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1641                                       /* op1 = variable, val.i = constant             */
1642
1643                         /* using inc and dec is definitely faster than add -- tested      */
1644                         /* with sieve                                                     */
1645
1646                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1647                         d = var->regoff;
1648                         if (var->flags & INMEMORY) {
1649                                 if (iptr->val.i == 1) {
1650                                         x86_64_incl_membase(cd, REG_SP, d * 8);
1651
1652                                 } else if (iptr->val.i == -1) {
1653                                         x86_64_decl_membase(cd, REG_SP, d * 8);
1654
1655                                 } else {
1656                                         x86_64_alul_imm_membase(cd, X86_64_ADD, iptr->val.i, REG_SP, d * 8);
1657                                 }
1658
1659                         } else {
1660                                 if (iptr->val.i == 1) {
1661                                         x86_64_incl_reg(cd, d);
1662
1663                                 } else if (iptr->val.i == -1) {
1664                                         x86_64_decl_reg(cd, d);
1665
1666                                 } else {
1667                                         x86_64_alul_imm_reg(cd, X86_64_ADD, iptr->val.i, d);
1668                                 }
1669                         }
1670                         break;
1671
1672
1673                 /* floating operations ************************************************/
1674
1675                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1676
1677                         var_to_reg_flt(s1, src, REG_FTMP1);
1678                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1679                         disp = dseg_adds4(cd, 0x80000000);
1680                         M_FLTMOVE(s1, d);
1681                         x86_64_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1682                         x86_64_xorps_reg_reg(cd, REG_FTMP2, d);
1683                         store_reg_to_var_flt(iptr->dst, d);
1684                         break;
1685
1686                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1687
1688                         var_to_reg_flt(s1, src, REG_FTMP1);
1689                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1690                         disp = dseg_adds8(cd, 0x8000000000000000);
1691                         M_FLTMOVE(s1, d);
1692                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1693                         x86_64_xorpd_reg_reg(cd, REG_FTMP2, d);
1694                         store_reg_to_var_flt(iptr->dst, d);
1695                         break;
1696
1697                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1698
1699                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1700                         var_to_reg_flt(s2, src, REG_FTMP2);
1701                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1702                         if (s1 == d) {
1703                                 x86_64_addss_reg_reg(cd, s2, d);
1704                         } else if (s2 == d) {
1705                                 x86_64_addss_reg_reg(cd, s1, d);
1706                         } else {
1707                                 M_FLTMOVE(s1, d);
1708                                 x86_64_addss_reg_reg(cd, s2, d);
1709                         }
1710                         store_reg_to_var_flt(iptr->dst, d);
1711                         break;
1712
1713                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1714
1715                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1716                         var_to_reg_flt(s2, src, REG_FTMP2);
1717                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1718                         if (s1 == d) {
1719                                 x86_64_addsd_reg_reg(cd, s2, d);
1720                         } else if (s2 == d) {
1721                                 x86_64_addsd_reg_reg(cd, s1, d);
1722                         } else {
1723                                 M_FLTMOVE(s1, d);
1724                                 x86_64_addsd_reg_reg(cd, s2, d);
1725                         }
1726                         store_reg_to_var_flt(iptr->dst, d);
1727                         break;
1728
1729                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1730
1731                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1732                         var_to_reg_flt(s2, src, REG_FTMP2);
1733                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1734                         if (s2 == d) {
1735                                 M_FLTMOVE(s2, REG_FTMP2);
1736                                 s2 = REG_FTMP2;
1737                         }
1738                         M_FLTMOVE(s1, d);
1739                         x86_64_subss_reg_reg(cd, s2, d);
1740                         store_reg_to_var_flt(iptr->dst, d);
1741                         break;
1742
1743                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1744
1745                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1746                         var_to_reg_flt(s2, src, REG_FTMP2);
1747                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1748                         if (s2 == d) {
1749                                 M_FLTMOVE(s2, REG_FTMP2);
1750                                 s2 = REG_FTMP2;
1751                         }
1752                         M_FLTMOVE(s1, d);
1753                         x86_64_subsd_reg_reg(cd, s2, d);
1754                         store_reg_to_var_flt(iptr->dst, d);
1755                         break;
1756
1757                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1758
1759                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1760                         var_to_reg_flt(s2, src, REG_FTMP2);
1761                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1762                         if (s1 == d) {
1763                                 x86_64_mulss_reg_reg(cd, s2, d);
1764                         } else if (s2 == d) {
1765                                 x86_64_mulss_reg_reg(cd, s1, d);
1766                         } else {
1767                                 M_FLTMOVE(s1, d);
1768                                 x86_64_mulss_reg_reg(cd, s2, d);
1769                         }
1770                         store_reg_to_var_flt(iptr->dst, d);
1771                         break;
1772
1773                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1774
1775                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1776                         var_to_reg_flt(s2, src, REG_FTMP2);
1777                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1778                         if (s1 == d) {
1779                                 x86_64_mulsd_reg_reg(cd, s2, d);
1780                         } else if (s2 == d) {
1781                                 x86_64_mulsd_reg_reg(cd, s1, d);
1782                         } else {
1783                                 M_FLTMOVE(s1, d);
1784                                 x86_64_mulsd_reg_reg(cd, s2, d);
1785                         }
1786                         store_reg_to_var_flt(iptr->dst, d);
1787                         break;
1788
1789                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1790
1791                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1792                         var_to_reg_flt(s2, src, REG_FTMP2);
1793                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1794                         if (s2 == d) {
1795                                 M_FLTMOVE(s2, REG_FTMP2);
1796                                 s2 = REG_FTMP2;
1797                         }
1798                         M_FLTMOVE(s1, d);
1799                         x86_64_divss_reg_reg(cd, s2, d);
1800                         store_reg_to_var_flt(iptr->dst, d);
1801                         break;
1802
1803                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1804
1805                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1806                         var_to_reg_flt(s2, src, REG_FTMP2);
1807                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1808                         if (s2 == d) {
1809                                 M_FLTMOVE(s2, REG_FTMP2);
1810                                 s2 = REG_FTMP2;
1811                         }
1812                         M_FLTMOVE(s1, d);
1813                         x86_64_divsd_reg_reg(cd, s2, d);
1814                         store_reg_to_var_flt(iptr->dst, d);
1815                         break;
1816
1817                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1818
1819                         var_to_reg_int(s1, src, REG_ITMP1);
1820                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1821                         x86_64_cvtsi2ss_reg_reg(cd, s1, d);
1822                         store_reg_to_var_flt(iptr->dst, d);
1823                         break;
1824
1825                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1826
1827                         var_to_reg_int(s1, src, REG_ITMP1);
1828                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1829                         x86_64_cvtsi2sd_reg_reg(cd, s1, d);
1830                         store_reg_to_var_flt(iptr->dst, d);
1831                         break;
1832
1833                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1834
1835                         var_to_reg_int(s1, src, REG_ITMP1);
1836                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1837                         x86_64_cvtsi2ssq_reg_reg(cd, s1, d);
1838                         store_reg_to_var_flt(iptr->dst, d);
1839                         break;
1840
1841                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1842
1843                         var_to_reg_int(s1, src, REG_ITMP1);
1844                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1845                         x86_64_cvtsi2sdq_reg_reg(cd, s1, d);
1846                         store_reg_to_var_flt(iptr->dst, d);
1847                         break;
1848
1849                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1850
1851                         var_to_reg_flt(s1, src, REG_FTMP1);
1852                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1853                         x86_64_cvttss2si_reg_reg(cd, s1, d);
1854                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1855                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1856                         x86_64_jcc(cd, X86_64_CC_NE, a);
1857                         M_FLTMOVE(s1, REG_FTMP1);
1858                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_f2i, REG_ITMP2);
1859                         x86_64_call_reg(cd, REG_ITMP2);
1860                         M_INTMOVE(REG_RESULT, d);
1861                         store_reg_to_var_int(iptr->dst, d);
1862                         break;
1863
1864                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1865
1866                         var_to_reg_flt(s1, src, REG_FTMP1);
1867                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1868                         x86_64_cvttsd2si_reg_reg(cd, s1, d);
1869                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1870                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1871                         x86_64_jcc(cd, X86_64_CC_NE, a);
1872                         M_FLTMOVE(s1, REG_FTMP1);
1873                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_d2i, REG_ITMP2);
1874                         x86_64_call_reg(cd, REG_ITMP2);
1875                         M_INTMOVE(REG_RESULT, d);
1876                         store_reg_to_var_int(iptr->dst, d);
1877                         break;
1878
1879                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1880
1881                         var_to_reg_flt(s1, src, REG_FTMP1);
1882                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1883                         x86_64_cvttss2siq_reg_reg(cd, s1, d);
1884                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1885                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1886                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1887                         x86_64_jcc(cd, X86_64_CC_NE, a);
1888                         M_FLTMOVE(s1, REG_FTMP1);
1889                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_f2l, REG_ITMP2);
1890                         x86_64_call_reg(cd, REG_ITMP2);
1891                         M_INTMOVE(REG_RESULT, d);
1892                         store_reg_to_var_int(iptr->dst, d);
1893                         break;
1894
1895                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1896
1897                         var_to_reg_flt(s1, src, REG_FTMP1);
1898                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1899                         x86_64_cvttsd2siq_reg_reg(cd, s1, d);
1900                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1901                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1902                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1903                         x86_64_jcc(cd, X86_64_CC_NE, a);
1904                         M_FLTMOVE(s1, REG_FTMP1);
1905                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_d2l, REG_ITMP2);
1906                         x86_64_call_reg(cd, REG_ITMP2);
1907                         M_INTMOVE(REG_RESULT, d);
1908                         store_reg_to_var_int(iptr->dst, d);
1909                         break;
1910
1911                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1912
1913                         var_to_reg_flt(s1, src, REG_FTMP1);
1914                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1915                         x86_64_cvtss2sd_reg_reg(cd, s1, d);
1916                         store_reg_to_var_flt(iptr->dst, d);
1917                         break;
1918
1919                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1920
1921                         var_to_reg_flt(s1, src, REG_FTMP1);
1922                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1923                         x86_64_cvtsd2ss_reg_reg(cd, s1, d);
1924                         store_reg_to_var_flt(iptr->dst, d);
1925                         break;
1926
1927                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1928                                           /* == => 0, < => 1, > => -1 */
1929
1930                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1931                         var_to_reg_flt(s2, src, REG_FTMP2);
1932                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1933                         M_CLR(d);
1934                         M_MOV_IMM(1, REG_ITMP1);
1935                         M_MOV_IMM(-1, REG_ITMP2);
1936                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1937                         M_CMOVB(REG_ITMP1, d);
1938                         M_CMOVA(REG_ITMP2, d);
1939                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1940                         store_reg_to_var_int(iptr->dst, d);
1941                         break;
1942
1943                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1944                                           /* == => 0, < => 1, > => -1 */
1945
1946                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1947                         var_to_reg_flt(s2, src, REG_FTMP2);
1948                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1949                         M_CLR(d);
1950                         M_MOV_IMM(1, REG_ITMP1);
1951                         M_MOV_IMM(-1, REG_ITMP2);
1952                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1953                         M_CMOVB(REG_ITMP1, d);
1954                         M_CMOVA(REG_ITMP2, d);
1955                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1956                         store_reg_to_var_int(iptr->dst, d);
1957                         break;
1958
1959                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1960                                           /* == => 0, < => 1, > => -1 */
1961
1962                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1963                         var_to_reg_flt(s2, src, REG_FTMP2);
1964                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1965                         M_CLR(d);
1966                         M_MOV_IMM(1, REG_ITMP1);
1967                         M_MOV_IMM(-1, REG_ITMP2);
1968                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1969                         M_CMOVB(REG_ITMP1, d);
1970                         M_CMOVA(REG_ITMP2, d);
1971                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1972                         store_reg_to_var_int(iptr->dst, d);
1973                         break;
1974
1975                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1976                                           /* == => 0, < => 1, > => -1 */
1977
1978                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1979                         var_to_reg_flt(s2, src, REG_FTMP2);
1980                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1981                         M_CLR(d);
1982                         M_MOV_IMM(1, REG_ITMP1);
1983                         M_MOV_IMM(-1, REG_ITMP2);
1984                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1985                         M_CMOVB(REG_ITMP1, d);
1986                         M_CMOVA(REG_ITMP2, d);
1987                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1988                         store_reg_to_var_int(iptr->dst, d);
1989                         break;
1990
1991
1992                 /* memory operations **************************************************/
1993
1994                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1995
1996                         var_to_reg_int(s1, src, REG_ITMP1);
1997                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1998                         gen_nullptr_check(s1);
1999                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
2000                         store_reg_to_var_int(iptr->dst, d);
2001                         break;
2002
2003                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2004
2005                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2006                         var_to_reg_int(s2, src, REG_ITMP2);
2007                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2008                         if (iptr->op1 == 0) {
2009                                 gen_nullptr_check(s1);
2010                                 gen_bound_check;
2011                         }
2012                         x86_64_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2013                         store_reg_to_var_int(iptr->dst, d);
2014                         break;
2015
2016                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2017
2018                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2019                         var_to_reg_int(s2, src, REG_ITMP2);
2020                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2021                         if (iptr->op1 == 0) {
2022                                 gen_nullptr_check(s1);
2023                                 gen_bound_check;
2024                         }
2025                         x86_64_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2026                         store_reg_to_var_int(iptr->dst, d);
2027                         break;
2028
2029                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2030
2031                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2032                         var_to_reg_int(s2, src, REG_ITMP2);
2033                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2034                         if (iptr->op1 == 0) {
2035                                 gen_nullptr_check(s1);
2036                                 gen_bound_check;
2037                         }
2038                         x86_64_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2039                         store_reg_to_var_int(iptr->dst, d);
2040                         break;
2041
2042                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
2043
2044                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2045                         var_to_reg_int(s2, src, REG_ITMP2);
2046                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2047                         if (iptr->op1 == 0) {
2048                                 gen_nullptr_check(s1);
2049                                 gen_bound_check;
2050                         }
2051                         x86_64_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
2052                         store_reg_to_var_int(iptr->dst, d);
2053                         break;
2054
2055                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
2056
2057                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2058                         var_to_reg_int(s2, src, REG_ITMP2);
2059                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2060                         if (iptr->op1 == 0) {
2061                                 gen_nullptr_check(s1);
2062                                 gen_bound_check;
2063                         }
2064                         x86_64_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
2065                         store_reg_to_var_int(iptr->dst, d);
2066                         break;
2067
2068                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
2069
2070                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2071                         var_to_reg_int(s2, src, REG_ITMP2);
2072                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2073                         if (iptr->op1 == 0) {
2074                                 gen_nullptr_check(s1);
2075                                 gen_bound_check;
2076                         }
2077                         x86_64_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2078                         store_reg_to_var_flt(iptr->dst, d);
2079                         break;
2080
2081                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2082
2083                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2084                         var_to_reg_int(s2, src, REG_ITMP2);
2085                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2086                         if (iptr->op1 == 0) {
2087                                 gen_nullptr_check(s1);
2088                                 gen_bound_check;
2089                         }
2090                         x86_64_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2091                         store_reg_to_var_flt(iptr->dst, d);
2092                         break;
2093
2094                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
2095
2096                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2097                         var_to_reg_int(s2, src, REG_ITMP2);
2098                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2099                         if (iptr->op1 == 0) {
2100                                 gen_nullptr_check(s1);
2101                                 gen_bound_check;
2102                         }
2103                         x86_64_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
2104                         store_reg_to_var_int(iptr->dst, d);
2105                         break;
2106
2107
2108                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2109
2110                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2111                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2112                         if (iptr->op1 == 0) {
2113                                 gen_nullptr_check(s1);
2114                                 gen_bound_check;
2115                         }
2116                         var_to_reg_int(s3, src, REG_ITMP3);
2117                         x86_64_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2118                         break;
2119
2120                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2121
2122                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2123                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2124                         if (iptr->op1 == 0) {
2125                                 gen_nullptr_check(s1);
2126                                 gen_bound_check;
2127                         }
2128                         var_to_reg_int(s3, src, REG_ITMP3);
2129                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2130                         break;
2131
2132                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2133
2134                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2135                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2136                         if (iptr->op1 == 0) {
2137                                 gen_nullptr_check(s1);
2138                                 gen_bound_check;
2139                         }
2140                         var_to_reg_int(s3, src, REG_ITMP3);
2141                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2142                         break;
2143
2144                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2145
2146                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2147                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2148                         if (iptr->op1 == 0) {
2149                                 gen_nullptr_check(s1);
2150                                 gen_bound_check;
2151                         }
2152                         var_to_reg_int(s3, src, REG_ITMP3);
2153                         x86_64_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2154                         break;
2155
2156                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2157
2158                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2159                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2160                         if (iptr->op1 == 0) {
2161                                 gen_nullptr_check(s1);
2162                                 gen_bound_check;
2163                         }
2164                         var_to_reg_int(s3, src, REG_ITMP3);
2165                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2166                         break;
2167
2168                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2169
2170                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2171                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2172                         if (iptr->op1 == 0) {
2173                                 gen_nullptr_check(s1);
2174                                 gen_bound_check;
2175                         }
2176                         var_to_reg_flt(s3, src, REG_FTMP3);
2177                         x86_64_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2178                         break;
2179
2180                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2181
2182                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2183                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2184                         if (iptr->op1 == 0) {
2185                                 gen_nullptr_check(s1);
2186                                 gen_bound_check;
2187                         }
2188                         var_to_reg_flt(s3, src, REG_FTMP3);
2189                         x86_64_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2190                         break;
2191
2192                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2193
2194                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2195                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2196                         if (iptr->op1 == 0) {
2197                                 gen_nullptr_check(s1);
2198                                 gen_bound_check;
2199                         }
2200                         var_to_reg_int(s3, src, REG_ITMP3);
2201
2202                         M_MOV(s1, rd->argintregs[0]);
2203                         M_MOV(s3, rd->argintregs[1]);
2204                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
2205                         M_CALL(REG_ITMP1);
2206                         M_TEST(REG_RESULT);
2207                         M_BEQ(0);
2208                         codegen_add_arraystoreexception_ref(cd, cd->mcodeptr);
2209
2210                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2211                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2212                         var_to_reg_int(s3, src, REG_ITMP3);
2213                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2214                         break;
2215
2216
2217                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2218
2219                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2220                         var_to_reg_int(s2, src, REG_ITMP2);
2221                         if (iptr->op1 == 0) {
2222                                 gen_nullptr_check(s1);
2223                                 gen_bound_check;
2224                         }
2225                         x86_64_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2226                         break;
2227
2228                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2229
2230                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2231                         var_to_reg_int(s2, src, REG_ITMP2);
2232                         if (iptr->op1 == 0) {
2233                                 gen_nullptr_check(s1);
2234                                 gen_bound_check;
2235                         }
2236                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2237                         break;
2238
2239                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2240
2241                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2242                         var_to_reg_int(s2, src, REG_ITMP2);
2243                         if (iptr->op1 == 0) {
2244                                 gen_nullptr_check(s1);
2245                                 gen_bound_check;
2246                         }
2247                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2248                         break;
2249
2250                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2251
2252                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2253                         var_to_reg_int(s2, src, REG_ITMP2);
2254                         if (iptr->op1 == 0) {
2255                                 gen_nullptr_check(s1);
2256                                 gen_bound_check;
2257                         }
2258                         x86_64_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2259                         break;
2260
2261                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2262
2263                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2264                         var_to_reg_int(s2, src, REG_ITMP2);
2265                         if (iptr->op1 == 0) {
2266                                 gen_nullptr_check(s1);
2267                                 gen_bound_check;
2268                         }
2269
2270                         if (IS_IMM32(iptr->val.l)) {
2271                                 x86_64_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2272                         } else {
2273                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2274                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2275                         }
2276                         break;
2277
2278                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2279
2280                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2281                         var_to_reg_int(s2, src, REG_ITMP2);
2282                         if (iptr->op1 == 0) {
2283                                 gen_nullptr_check(s1);
2284                                 gen_bound_check;
2285                         }
2286                         x86_64_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2287                         break;
2288
2289
2290                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2291                                       /* op1 = type, val.a = field address            */
2292
2293                         if (iptr->val.a == NULL) {
2294                                 disp = dseg_addaddress(cd, NULL);
2295
2296 /*                              PROFILE_CYCLE_STOP; */
2297
2298                                 codegen_addpatchref(cd, cd->mcodeptr,
2299                                                                         PATCHER_get_putstatic,
2300                                                                         (unresolved_field *) iptr->target, disp);
2301
2302                                 if (opt_showdisassemble) {
2303                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2304                                 }
2305
2306 /*                              PROFILE_CYCLE_START; */
2307
2308                         } else {
2309                                 fieldinfo *fi = iptr->val.a;
2310
2311                                 disp = dseg_addaddress(cd, &(fi->value));
2312
2313                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2314                                         PROFILE_CYCLE_STOP;
2315
2316                                         codegen_addpatchref(cd, cd->mcodeptr,
2317                                                                                 PATCHER_clinit, fi->class, 0);
2318
2319                                         if (opt_showdisassemble) {
2320                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2321                                         }
2322
2323                                         PROFILE_CYCLE_START;
2324                                 }
2325                         }
2326
2327                         /* This approach is much faster than moving the field
2328                            address inline into a register. */
2329
2330                         M_ALD(REG_ITMP2, RIP, -(((ptrint) cd->mcodeptr + 7) -
2331                                                                         (ptrint) cd->mcodebase) + disp);
2332
2333                         switch (iptr->op1) {
2334                         case TYPE_INT:
2335                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2336                                 M_ILD(d, REG_ITMP2, 0);
2337                                 store_reg_to_var_int(iptr->dst, d);
2338                                 break;
2339                         case TYPE_LNG:
2340                         case TYPE_ADR:
2341                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2342                                 M_LLD(d, REG_ITMP2, 0);
2343                                 store_reg_to_var_int(iptr->dst, d);
2344                                 break;
2345                         case TYPE_FLT:
2346                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2347                                 x86_64_movss_membase_reg(cd, REG_ITMP2, 0, d);
2348                                 store_reg_to_var_flt(iptr->dst, d);
2349                                 break;
2350                         case TYPE_DBL:
2351                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2352                                 x86_64_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2353                                 store_reg_to_var_flt(iptr->dst, d);
2354                                 break;
2355                         }
2356                         break;
2357
2358                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2359                                       /* op1 = type, val.a = field address            */
2360
2361                         if (iptr->val.a == NULL) {
2362                                 disp = dseg_addaddress(cd, NULL);
2363
2364 /*                              PROFILE_CYCLE_STOP; */
2365
2366                                 codegen_addpatchref(cd, cd->mcodeptr,
2367                                                                         PATCHER_get_putstatic,
2368                                                                         (unresolved_field *) iptr->target, disp);
2369
2370                                 if (opt_showdisassemble) {
2371                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2372                                 }
2373
2374 /*                              PROFILE_CYCLE_START; */
2375
2376                         } else {
2377                                 fieldinfo *fi = iptr->val.a;
2378
2379                                 disp = dseg_addaddress(cd, &(fi->value));
2380
2381                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2382                                         PROFILE_CYCLE_STOP;
2383
2384                                         codegen_addpatchref(cd, cd->mcodeptr,
2385                                                                                 PATCHER_clinit, fi->class, 0);
2386
2387                                         if (opt_showdisassemble) {
2388                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2389                                         }
2390
2391                                         PROFILE_CYCLE_START;
2392                                 }
2393                         }
2394
2395                         /* This approach is much faster than moving the field
2396                            address inline into a register. */
2397
2398                         M_ALD(REG_ITMP2, RIP, -(((ptrint) cd->mcodeptr + 7) -
2399                                                                         (ptrint) cd->mcodebase) + disp);
2400
2401                         switch (iptr->op1) {
2402                         case TYPE_INT:
2403                                 var_to_reg_int(s2, src, REG_ITMP1);
2404                                 M_IST(s2, REG_ITMP2, 0);
2405                                 break;
2406                         case TYPE_LNG:
2407                         case TYPE_ADR:
2408                                 var_to_reg_int(s2, src, REG_ITMP1);
2409                                 M_LST(s2, REG_ITMP2, 0);
2410                                 break;
2411                         case TYPE_FLT:
2412                                 var_to_reg_flt(s2, src, REG_FTMP1);
2413                                 x86_64_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2414                                 break;
2415                         case TYPE_DBL:
2416                                 var_to_reg_flt(s2, src, REG_FTMP1);
2417                                 x86_64_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2418                                 break;
2419                         }
2420                         break;
2421
2422                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2423                                           /* val = value (in current instruction)     */
2424                                           /* op1 = type, val.a = field address (in    */
2425                                           /* following NOP)                           */
2426
2427                         if (iptr[1].val.a == NULL) {
2428                                 disp = dseg_addaddress(cd, NULL);
2429
2430 /*                              PROFILE_CYCLE_STOP; */
2431
2432                                 codegen_addpatchref(cd, cd->mcodeptr,
2433                                                                         PATCHER_get_putstatic,
2434                                                                         (unresolved_field *) iptr[1].target, disp);
2435
2436                                 if (opt_showdisassemble) {
2437                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2438                                 }
2439
2440 /*                              PROFILE_CYCLE_START; */
2441
2442                         } else {
2443                                 fieldinfo *fi = iptr[1].val.a;
2444
2445                                 disp = dseg_addaddress(cd, &(fi->value));
2446
2447                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2448                                         PROFILE_CYCLE_STOP;
2449
2450                                         codegen_addpatchref(cd, cd->mcodeptr,
2451                                                                                 PATCHER_clinit, fi->class, 0);
2452
2453                                         if (opt_showdisassemble) {
2454                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2455                                         }
2456
2457                                         PROFILE_CYCLE_START;
2458                                 }
2459                         }
2460
2461                         /* This approach is much faster than moving the field
2462                            address inline into a register. */
2463
2464                         M_ALD(REG_ITMP1, RIP, -(((ptrint) cd->mcodeptr + 7) -
2465                                                                         (ptrint) cd->mcodebase) + disp);
2466
2467                         switch (iptr->op1) {
2468                         case TYPE_INT:
2469                         case TYPE_FLT:
2470                                 M_IST_IMM(iptr->val.i, REG_ITMP1, 0);
2471                                 break;
2472                         case TYPE_LNG:
2473                         case TYPE_ADR:
2474                         case TYPE_DBL:
2475                                 if (IS_IMM32(iptr->val.l)) {
2476                                         M_LST_IMM32(iptr->val.l, REG_ITMP1, 0);
2477                                 } else {
2478                                         M_IST_IMM(iptr->val.l, REG_ITMP1, 0);
2479                                         M_IST_IMM(iptr->val.l >> 32, REG_ITMP1, 4);
2480                                 }
2481                                 break;
2482                         }
2483                         break;
2484
2485                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2486                                       /* op1 = type, val.i = field offset             */
2487
2488                         var_to_reg_int(s1, src, REG_ITMP1);
2489                         gen_nullptr_check(s1);
2490
2491                         if (iptr->val.a == NULL) {
2492 /*                              PROFILE_CYCLE_STOP; */
2493
2494                                 codegen_addpatchref(cd, cd->mcodeptr,
2495                                                                         PATCHER_get_putfield,
2496                                                                         (unresolved_field *) iptr->target, 0);
2497
2498                                 if (opt_showdisassemble) {
2499                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2500                                 }
2501
2502 /*                              PROFILE_CYCLE_START; */
2503
2504                                 disp = 0;
2505
2506                         } else {
2507                                 disp = ((fieldinfo *) (iptr->val.a))->offset;
2508                         }
2509
2510                         switch (iptr->op1) {
2511                         case TYPE_INT:
2512                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2513                                 if (iptr->val.a == NULL)
2514                                         M_ILD32(d, s1, disp);
2515                                 else
2516                                         M_ILD(d, s1, disp);
2517                                 store_reg_to_var_int(iptr->dst, d);
2518                                 break;
2519                         case TYPE_LNG:
2520                         case TYPE_ADR:
2521                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2522                                 if (iptr->val.a == NULL)
2523                                         M_LLD32(d, s1, disp);
2524                                 else
2525                                         M_LLD(d, s1, disp);
2526                                 store_reg_to_var_int(iptr->dst, d);
2527                                 break;
2528                         case TYPE_FLT:
2529                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2530                                 x86_64_movss_membase32_reg(cd, s1, disp, d);
2531                                 store_reg_to_var_flt(iptr->dst, d);
2532                                 break;
2533                         case TYPE_DBL:
2534                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2535                                 x86_64_movsd_membase32_reg(cd, s1, disp, d);
2536                                 store_reg_to_var_flt(iptr->dst, d);
2537                                 break;
2538                         }
2539                         break;
2540
2541                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2542                                       /* op1 = type, val.i = field offset             */
2543
2544                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2545                         gen_nullptr_check(s1);
2546
2547                         if (IS_INT_LNG_TYPE(iptr->op1)) {
2548                                 var_to_reg_int(s2, src, REG_ITMP2);
2549                         } else {
2550                                 var_to_reg_flt(s2, src, REG_FTMP2);
2551                         }
2552
2553                         if (iptr->val.a == NULL) {
2554 /*                              PROFILE_CYCLE_STOP; */
2555
2556                                 codegen_addpatchref(cd, cd->mcodeptr,
2557                                                                         PATCHER_get_putfield,
2558                                                                         (unresolved_field *) iptr->target, 0);
2559
2560                                 if (opt_showdisassemble) {
2561                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2562                                 }
2563
2564 /*                              PROFILE_CYCLE_START; */
2565
2566                                 disp = 0;
2567
2568                         } else {
2569                                 disp = ((fieldinfo *) (iptr->val.a))->offset;
2570                         }
2571
2572                         switch (iptr->op1) {
2573                         case TYPE_INT:
2574                                 if (iptr->val.a == NULL)
2575                                         M_IST32(s2, s1, disp);
2576                                 else
2577                                         M_IST(s2, s1, disp);
2578                                 break;
2579                         case TYPE_LNG:
2580                         case TYPE_ADR:
2581                                 if (iptr->val.a == NULL)
2582                                         M_LST32(s2, s1, disp);
2583                                 else
2584                                         M_LST(s2, s1, disp);
2585                                 break;
2586                         case TYPE_FLT:
2587                                 x86_64_movss_reg_membase32(cd, s2, s1, disp);
2588                                 break;
2589                         case TYPE_DBL:
2590                                 x86_64_movsd_reg_membase32(cd, s2, s1, disp);
2591                                 break;
2592                         }
2593                         break;
2594
2595                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2596                                           /* val = value (in current instruction)     */
2597                                           /* op1 = type, val.a = field address (in    */
2598                                           /* following NOP)                           */
2599
2600                         var_to_reg_int(s1, src, REG_ITMP1);
2601                         gen_nullptr_check(s1);
2602
2603                         if (iptr[1].val.a == NULL) {
2604 /*                              PROFILE_CYCLE_STOP; */
2605
2606                                 codegen_addpatchref(cd, cd->mcodeptr,
2607                                                                         PATCHER_putfieldconst,
2608                                                                         (unresolved_field *) iptr[1].target, 0);
2609
2610                                 if (opt_showdisassemble) {
2611                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2612                                 }
2613
2614 /*                              PROFILE_CYCLE_START; */
2615
2616                                 disp = 0;
2617
2618                         } else {
2619                                 disp = ((fieldinfo *) (iptr[1].val.a))->offset;
2620                         }
2621
2622                         switch (iptr->op1) {
2623                         case TYPE_INT:
2624                         case TYPE_FLT:
2625                                 if (iptr[1].val.a == NULL)
2626                                         M_IST32_IMM(iptr->val.i, s1, disp);
2627                                 else
2628                                         M_IST_IMM(iptr->val.i, s1, disp);
2629                                 break;
2630                         case TYPE_LNG:
2631                         case TYPE_ADR:
2632                         case TYPE_DBL:
2633                                 /* We can only optimize the move, if the class is
2634                                    resolved. Otherwise we don't know what to patch. */
2635                                 if (iptr[1].val.a == NULL) {
2636                                         M_IST32_IMM(iptr->val.l, s1, disp);
2637                                         M_IST32_IMM(iptr->val.l >> 32, s1, disp + 4);
2638                                 } else {
2639                                         if (IS_IMM32(iptr->val.l)) {
2640                                                 M_LST_IMM32(iptr->val.l, s1, disp);
2641                                         } else {
2642                                                 M_IST_IMM(iptr->val.l, s1, disp);
2643                                                 M_IST_IMM(iptr->val.l >> 32, s1, disp + 4);
2644                                         }
2645                                 }
2646                                 break;
2647                         }
2648                         break;
2649
2650
2651                 /* branch operations **************************************************/
2652
2653                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2654
2655                         var_to_reg_int(s1, src, REG_ITMP1);
2656                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2657
2658                         PROFILE_CYCLE_STOP;
2659
2660 #ifdef ENABLE_VERIFIER
2661                         if (iptr->val.a) {
2662                                 codegen_addpatchref(cd, cd->mcodeptr,
2663                                                                         PATCHER_athrow_areturn,
2664                                                                         (unresolved_class *) iptr->val.a, 0);
2665
2666                                 if (opt_showdisassemble) {
2667                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2668                                 }
2669                         }
2670 #endif /* ENABLE_VERIFIER */
2671
2672                         M_CALL_IMM(0);                            /* passing exception pc */
2673                         M_POP(REG_ITMP2_XPC);
2674
2675                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2676                         M_JMP(REG_ITMP3);
2677                         break;
2678
2679                 case ICMD_GOTO:         /* ... ==> ...                                */
2680                                         /* op1 = target JavaVM pc                     */
2681
2682                         M_JMP_IMM(0);
2683                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2684                         break;
2685
2686                 case ICMD_JSR:          /* ... ==> ...                                */
2687                                         /* op1 = target JavaVM pc                     */
2688
2689                         M_CALL_IMM(0);
2690                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2691                         break;
2692
2693                 case ICMD_RET:          /* ... ==> ...                                */
2694                                         /* op1 = local variable                       */
2695
2696                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2697                         var_to_reg_int(s1, var, REG_ITMP1);
2698                         M_JMP(s1);
2699                         break;
2700
2701                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2702                                         /* op1 = target JavaVM pc                     */
2703
2704                         if (src->flags & INMEMORY)
2705                                 M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
2706                         else
2707                                 M_TEST(src->regoff);
2708                         M_BEQ(0);
2709                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2710                         break;
2711
2712                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2713                                         /* op1 = target JavaVM pc                     */
2714
2715                         if (src->flags & INMEMORY)
2716                                 M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
2717                         else
2718                                 M_TEST(src->regoff);
2719                         M_BNE(0);
2720                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2721                         break;
2722
2723                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2724                                         /* op1 = target JavaVM pc, val.i = constant   */
2725
2726                         x86_64_emit_ifcc(cd, X86_64_CC_E, src, iptr);
2727                         break;
2728
2729                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2730                                         /* op1 = target JavaVM pc, val.i = constant   */
2731
2732                         x86_64_emit_ifcc(cd, X86_64_CC_L, src, iptr);
2733                         break;
2734
2735                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2736                                         /* op1 = target JavaVM pc, val.i = constant   */
2737
2738                         x86_64_emit_ifcc(cd, X86_64_CC_LE, src, iptr);
2739                         break;
2740
2741                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2742                                         /* op1 = target JavaVM pc, val.i = constant   */
2743
2744                         x86_64_emit_ifcc(cd, X86_64_CC_NE, src, iptr);
2745                         break;
2746
2747                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2748                                         /* op1 = target JavaVM pc, val.i = constant   */
2749
2750                         x86_64_emit_ifcc(cd, X86_64_CC_G, src, iptr);
2751                         break;
2752
2753                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2754                                         /* op1 = target JavaVM pc, val.i = constant   */
2755
2756                         x86_64_emit_ifcc(cd, X86_64_CC_GE, src, iptr);
2757                         break;
2758
2759                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2760                                         /* op1 = target JavaVM pc, val.l = constant   */
2761
2762                         x86_64_emit_if_lcc(cd, X86_64_CC_E, src, iptr);
2763                         break;
2764
2765                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2766                                         /* op1 = target JavaVM pc, val.l = constant   */
2767
2768                         x86_64_emit_if_lcc(cd, X86_64_CC_L, src, iptr);
2769                         break;
2770
2771                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2772                                         /* op1 = target JavaVM pc, val.l = constant   */
2773
2774                         x86_64_emit_if_lcc(cd, X86_64_CC_LE, src, iptr);
2775                         break;
2776
2777                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2778                                         /* op1 = target JavaVM pc, val.l = constant   */
2779
2780                         x86_64_emit_if_lcc(cd, X86_64_CC_NE, src, iptr);
2781                         break;
2782
2783                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2784                                         /* op1 = target JavaVM pc, val.l = constant   */
2785
2786                         x86_64_emit_if_lcc(cd, X86_64_CC_G, src, iptr);
2787                         break;
2788
2789                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2790                                         /* op1 = target JavaVM pc, val.l = constant   */
2791
2792                         x86_64_emit_if_lcc(cd, X86_64_CC_GE, src, iptr);
2793                         break;
2794
2795                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2796                                         /* op1 = target JavaVM pc                     */
2797
2798                         x86_64_emit_if_icmpcc(cd, X86_64_CC_E, src, iptr);
2799                         break;
2800
2801                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2802                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2803
2804                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_E, src, iptr);
2805                         break;
2806
2807                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2808                                         /* op1 = target JavaVM pc                     */
2809
2810                         x86_64_emit_if_icmpcc(cd, X86_64_CC_NE, src, iptr);
2811                         break;
2812
2813                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2814                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2815
2816                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_NE, src, iptr);
2817                         break;
2818
2819                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2820                                         /* op1 = target JavaVM pc                     */
2821
2822                         x86_64_emit_if_icmpcc(cd, X86_64_CC_L, src, iptr);
2823                         break;
2824
2825                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2826                                     /* op1 = target JavaVM pc                     */
2827
2828                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_L, src, iptr);
2829                         break;
2830
2831                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2832                                         /* op1 = target JavaVM pc                     */
2833
2834                         x86_64_emit_if_icmpcc(cd, X86_64_CC_G, src, iptr);
2835                         break;
2836
2837                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2838                                 /* op1 = target JavaVM pc                     */
2839
2840                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_G, src, iptr);
2841                         break;
2842
2843                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2844                                         /* op1 = target JavaVM pc                     */
2845
2846                         x86_64_emit_if_icmpcc(cd, X86_64_CC_LE, src, iptr);
2847                         break;
2848
2849                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2850                                         /* op1 = target JavaVM pc                     */
2851
2852                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_LE, src, iptr);
2853                         break;
2854
2855                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2856                                         /* op1 = target JavaVM pc                     */
2857
2858                         x86_64_emit_if_icmpcc(cd, X86_64_CC_GE, src, iptr);
2859                         break;
2860
2861                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2862                                     /* op1 = target JavaVM pc                     */
2863
2864                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_GE, src, iptr);
2865                         break;
2866
2867                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2868
2869                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2870                         break;
2871
2872                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2873                 case ICMD_IFNE_ICONST:  /* val.i = constant                           */
2874                 case ICMD_IFLT_ICONST:
2875                 case ICMD_IFGE_ICONST:
2876                 case ICMD_IFGT_ICONST:
2877                 case ICMD_IFLE_ICONST:
2878
2879                         var_to_reg_int(s1, src, REG_ITMP1);
2880                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2881                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2882                                 if (s1 == d) {
2883                                         M_INTMOVE(s1, REG_ITMP1);
2884                                         s1 = REG_ITMP1;
2885                                 }
2886                                 if (iptr[1].val.i == 0)
2887                                         M_CLR(d);
2888                                 else
2889                                         M_IMOV_IMM(iptr[1].val.i, d);
2890                         }
2891                         if (iptr->val.i == 0)
2892                                 M_CLR(REG_ITMP2);
2893                         else
2894                                 M_IMOV_IMM(iptr->val.i, REG_ITMP2);
2895                         M_ITEST(s1);
2896
2897                         switch (iptr->opc) {
2898                         case ICMD_IFEQ_ICONST:
2899                                 M_CMOVEQ(REG_ITMP2, d);
2900                                 break;
2901                         case ICMD_IFNE_ICONST:
2902                                 M_CMOVNE(REG_ITMP2, d);
2903                                 break;
2904                         case ICMD_IFLT_ICONST:
2905                                 M_CMOVLT(REG_ITMP2, d);
2906                                 break;
2907                         case ICMD_IFGE_ICONST:
2908                                 M_CMOVGE(REG_ITMP2, d);
2909                                 break;
2910                         case ICMD_IFGT_ICONST:
2911                                 M_CMOVGT(REG_ITMP2, d);
2912                                 break;
2913                         case ICMD_IFLE_ICONST:
2914                                 M_CMOVLE(REG_ITMP2, d);
2915                                 break;
2916                         }
2917
2918                         store_reg_to_var_int(iptr->dst, d);
2919                         break;
2920
2921
2922                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2923                 case ICMD_LRETURN:
2924
2925                         var_to_reg_int(s1, src, REG_RESULT);
2926                         M_INTMOVE(s1, REG_RESULT);
2927                         goto nowperformreturn;
2928
2929                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2930
2931                         var_to_reg_int(s1, src, REG_RESULT);
2932                         M_INTMOVE(s1, REG_RESULT);
2933
2934 #ifdef ENABLE_VERIFIER
2935                         if (iptr->val.a) {
2936                                 PROFILE_CYCLE_STOP;
2937
2938                                 codegen_addpatchref(cd, cd->mcodeptr,
2939                                                                         PATCHER_athrow_areturn,
2940                                                                         (unresolved_class *) iptr->val.a, 0);
2941
2942                                 if (opt_showdisassemble) {
2943                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2944                                 }
2945
2946                                 PROFILE_CYCLE_START;
2947                         }
2948 #endif /* ENABLE_VERIFIER */
2949                         goto nowperformreturn;
2950
2951                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2952                 case ICMD_DRETURN:
2953
2954                         var_to_reg_flt(s1, src, REG_FRESULT);
2955                         M_FLTMOVE(s1, REG_FRESULT);
2956                         goto nowperformreturn;
2957
2958                 case ICMD_RETURN:      /* ...  ==> ...                                */
2959
2960 nowperformreturn:
2961                         {
2962                         s4 i, p;
2963
2964                         p = parentargs_base;
2965
2966                         /* call trace function */
2967                         if (opt_verbosecall) {
2968                                 x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
2969
2970                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2971                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2972
2973                                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
2974                                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2975                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2976                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2977
2978                                 x86_64_mov_imm_reg(cd, (u8) builtin_displaymethodstop, REG_ITMP1);
2979                                 x86_64_call_reg(cd, REG_ITMP1);
2980
2981                                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2982                                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2983
2984                                 x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
2985                         }
2986
2987 #if defined(USE_THREADS)
2988                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2989                                 M_ALD(rd->argintregs[0], REG_SP, rd->memuse * 8);
2990
2991                                 /* we need to save the proper return value */
2992                                 switch (iptr->opc) {
2993                                 case ICMD_IRETURN:
2994                                 case ICMD_ARETURN:
2995                                 case ICMD_LRETURN:
2996                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2997                                         break;
2998                                 case ICMD_FRETURN:
2999                                 case ICMD_DRETURN:
3000                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
3001                                         break;
3002                                 }
3003
3004                                 M_MOV_IMM(builtin_monitorexit, REG_ITMP1);
3005                                 M_CALL(REG_ITMP1);
3006
3007                                 /* and now restore the proper return value */
3008                                 switch (iptr->opc) {
3009                                 case ICMD_IRETURN:
3010                                 case ICMD_ARETURN:
3011                                 case ICMD_LRETURN:
3012                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
3013                                         break;
3014                                 case ICMD_FRETURN:
3015                                 case ICMD_DRETURN:
3016                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
3017                                         break;
3018                                 }
3019                         }
3020 #endif
3021
3022                         /* restore saved registers */
3023
3024                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
3025                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
3026                         }
3027                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
3028                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
3029                         }
3030
3031                         /* deallocate stack */
3032
3033                         if (parentargs_base)
3034                                 M_AADD_IMM(parentargs_base * 8, REG_SP);
3035
3036                         /* generate method profiling code */
3037
3038                         PROFILE_CYCLE_STOP;
3039
3040                         M_RET;
3041                         }
3042                         break;
3043
3044
3045                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
3046                         {
3047                                 s4 i, l, *s4ptr;
3048                                 void **tptr;
3049
3050                                 tptr = (void **) iptr->target;
3051
3052                                 s4ptr = iptr->val.a;
3053                                 l = s4ptr[1];                          /* low     */
3054                                 i = s4ptr[2];                          /* high    */
3055
3056                                 var_to_reg_int(s1, src, REG_ITMP1);
3057                                 M_INTMOVE(s1, REG_ITMP1);
3058                                 if (l != 0) {
3059                                         x86_64_alul_imm_reg(cd, X86_64_SUB, l, REG_ITMP1);
3060                                 }
3061                                 i = i - l + 1;
3062
3063                 /* range check */
3064                                 x86_64_alul_imm_reg(cd, X86_64_CMP, i - 1, REG_ITMP1);
3065                                 x86_64_jcc(cd, X86_64_CC_A, 0);
3066
3067                 /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[0]), cd->mcodeptr); */
3068                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3069
3070                                 /* build jump table top down and use address of lowest entry */
3071
3072                 /* s4ptr += 3 + i; */
3073                                 tptr += i;
3074
3075                                 while (--i >= 0) {
3076                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
3077                                         --tptr;
3078                                 }
3079
3080                                 /* length of dataseg after last dseg_addtarget is used by load */
3081
3082                                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2);
3083                                 dseg_adddata(cd, cd->mcodeptr);
3084                                 x86_64_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
3085                                 x86_64_jmp_reg(cd, REG_ITMP1);
3086                         }
3087                         break;
3088
3089
3090                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
3091                         {
3092                                 s4 i, l, val, *s4ptr;
3093                                 void **tptr;
3094
3095                                 tptr = (void **) iptr->target;
3096
3097                                 s4ptr = iptr->val.a;
3098                                 l = s4ptr[0];                          /* default  */
3099                                 i = s4ptr[1];                          /* count    */
3100
3101                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
3102                                 var_to_reg_int(s1, src, REG_ITMP1);    /* reg compare should always be faster */
3103                                 while (--i >= 0) {
3104                                         s4ptr += 2;
3105                                         ++tptr;
3106
3107                                         val = s4ptr[0];
3108                                         x86_64_alul_imm_reg(cd, X86_64_CMP, val, s1);
3109                                         x86_64_jcc(cd, X86_64_CC_E, 0);
3110                                         codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3111                                 }
3112
3113                                 x86_64_jmp_imm(cd, 0);
3114
3115                                 tptr = (void **) iptr->target;
3116                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3117                         }
3118                         break;
3119
3120
3121                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
3122                                         /* op1 = arg count val.a = builtintable entry */
3123
3124                         bte = iptr->val.a;
3125                         md = bte->md;
3126                         goto gen_method;
3127
3128                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
3129                                         /* op1 = arg count, val.a = method pointer    */
3130
3131                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
3132                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
3133                 case ICMD_INVOKEINTERFACE:
3134
3135                         lm = iptr->val.a;
3136
3137                         if (lm == NULL) {
3138                                 unresolved_method *um = iptr->target;
3139                                 md = um->methodref->parseddesc.md;
3140                         } else {
3141                                 md = lm->parseddesc;
3142                         }
3143
3144 gen_method:
3145                         s3 = md->paramcount;
3146
3147                         MCODECHECK((20 * s3) + 128);
3148
3149                         /* copy arguments to registers or stack location */
3150
3151                         for (s3 = s3 - 1; s3 >= 0; s3--, src = src->prev) {
3152                                 if (src->varkind == ARGVAR)
3153                                         continue;
3154                                 if (IS_INT_LNG_TYPE(src->type)) {
3155                                         if (!md->params[s3].inmemory) {
3156                                                 s1 = rd->argintregs[md->params[s3].regoff];
3157                                                 var_to_reg_int(d, src, s1);
3158                                                 M_INTMOVE(d, s1);
3159                                         } else {
3160                                                 var_to_reg_int(d, src, REG_ITMP1);
3161                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
3162                                         }
3163
3164                                 } else {
3165                                         if (!md->params[s3].inmemory) {
3166                                                 s1 = rd->argfltregs[md->params[s3].regoff];
3167                                                 var_to_reg_flt(d, src, s1);
3168                                                 M_FLTMOVE(d, s1);
3169                                         } else {
3170                                                 var_to_reg_flt(d, src, REG_FTMP1);
3171                                                 M_DST(d, REG_SP, md->params[s3].regoff * 8);
3172                                         }
3173                                 }
3174                         }
3175
3176                         /* generate method profiling code */
3177
3178                         PROFILE_CYCLE_STOP;
3179
3180                         switch (iptr->opc) {
3181                         case ICMD_BUILTIN:
3182                                 a = (ptrint) bte->fp;
3183                                 d = md->returntype.type;
3184
3185                                 M_MOV_IMM(a, REG_ITMP1);
3186                                 M_CALL(REG_ITMP1);
3187
3188                                 /* if op1 == true, we need to check for an exception */
3189
3190                                 if (iptr->op1 == true) {
3191                                         M_TEST(REG_RESULT);
3192                                         M_BEQ(0);
3193                                         codegen_add_fillinstacktrace_ref(cd, cd->mcodeptr);
3194                                 }
3195                                 break;
3196
3197                         case ICMD_INVOKESPECIAL:
3198                                 M_TEST(rd->argintregs[0]);
3199                                 M_BEQ(0);
3200                                 codegen_add_nullpointerexception_ref(cd, cd->mcodeptr);
3201
3202                                 /* first argument contains pointer */
3203 /*                              gen_nullptr_check(rd->argintregs[0]); */
3204
3205                                 /* access memory for hardware nullptr */
3206 /*                              x86_64_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); */
3207
3208                                 /* fall through */
3209
3210                         case ICMD_INVOKESTATIC:
3211                                 if (lm == NULL) {
3212                                         unresolved_method *um = iptr->target;
3213
3214                                         codegen_addpatchref(cd, cd->mcodeptr,
3215                                                                                 PATCHER_invokestatic_special, um, 0);
3216
3217                                         if (opt_showdisassemble) {
3218                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3219                                         }
3220
3221                                         a = 0;
3222                                         d = um->methodref->parseddesc.md->returntype.type;
3223
3224                                 } else {
3225                                         a = (ptrint) lm->stubroutine;
3226                                         d = lm->parseddesc->returntype.type;
3227                                 }
3228
3229                                 M_MOV_IMM(a, REG_ITMP2);
3230                                 M_CALL(REG_ITMP2);
3231                                 break;
3232
3233                         case ICMD_INVOKEVIRTUAL:
3234                                 gen_nullptr_check(rd->argintregs[0]);
3235
3236                                 if (lm == NULL) {
3237                                         unresolved_method *um = iptr->target;
3238
3239                                         codegen_addpatchref(cd, cd->mcodeptr,
3240                                                                                 PATCHER_invokevirtual, um, 0);
3241
3242                                         if (opt_showdisassemble) {
3243                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3244                                         }
3245
3246                                         s1 = 0;
3247                                         d = um->methodref->parseddesc.md->returntype.type;
3248
3249                                 } else {
3250                                         s1 = OFFSET(vftbl_t, table[0]) +
3251                                                 sizeof(methodptr) * lm->vftblindex;
3252                                         d = lm->parseddesc->returntype.type;
3253                                 }
3254
3255                                 x86_64_mov_membase_reg(cd, rd->argintregs[0],
3256                                                                            OFFSET(java_objectheader, vftbl),
3257                                                                            REG_ITMP2);
3258                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP1);
3259                                 M_CALL(REG_ITMP1);
3260                                 break;
3261
3262                         case ICMD_INVOKEINTERFACE:
3263                                 gen_nullptr_check(rd->argintregs[0]);
3264
3265                                 if (lm == NULL) {
3266                                         unresolved_method *um = iptr->target;
3267
3268                                         codegen_addpatchref(cd, cd->mcodeptr,
3269                                                                                 PATCHER_invokeinterface, um, 0);
3270
3271                                         if (opt_showdisassemble) {
3272                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3273                                         }
3274
3275                                         s1 = 0;
3276                                         s2 = 0;
3277                                         d = um->methodref->parseddesc.md->returntype.type;
3278
3279                                 } else {
3280                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3281                                                 sizeof(methodptr) * lm->class->index;
3282
3283                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3284
3285                                         d = lm->parseddesc->returntype.type;
3286                                 }
3287
3288                                 M_ALD(REG_ITMP2, rd->argintregs[0],
3289                                           OFFSET(java_objectheader, vftbl));
3290                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP2);
3291                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s2, REG_ITMP1);
3292                                 M_CALL(REG_ITMP1);
3293                                 break;
3294                         }
3295
3296                         /* generate method profiling code */
3297
3298                         PROFILE_CYCLE_START;
3299
3300                         /* d contains return type */
3301
3302                         if (d != TYPE_VOID) {
3303                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3304                                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3305                                         M_INTMOVE(REG_RESULT, s1);
3306                                         store_reg_to_var_int(iptr->dst, s1);
3307                                 } else {
3308                                         s1 = reg_of_var(rd, iptr->dst, REG_FRESULT);
3309                                         M_FLTMOVE(REG_FRESULT, s1);
3310                                         store_reg_to_var_flt(iptr->dst, s1);
3311                                 }
3312                         }
3313                         break;
3314
3315
3316                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3317
3318                                       /* op1:   0 == array, 1 == class                */
3319                                       /* val.a: (classinfo *) superclass              */
3320
3321                         /*  superclass is an interface:
3322                          *
3323                          *  OK if ((sub == NULL) ||
3324                          *         (sub->vftbl->interfacetablelength > super->index) &&
3325                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3326                          *
3327                          *  superclass is a class:
3328                          *
3329                          *  OK if ((sub == NULL) || (0
3330                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3331                          *         super->vftbl->diffval));
3332                          */
3333
3334                         if (iptr->op1 == 1) {
3335                                 /* object type cast-check */
3336
3337                                 classinfo *super;
3338                                 vftbl_t   *supervftbl;
3339                                 s4         superindex;
3340
3341                                 super = (classinfo *) iptr->val.a;
3342
3343                                 if (!super) {
3344                                         superindex = 0;
3345                                         supervftbl = NULL;
3346
3347                                 } else {
3348                                         superindex = super->index;
3349                                         supervftbl = super->vftbl;
3350                                 }
3351
3352 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3353                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3354 #endif
3355                                 var_to_reg_int(s1, src, REG_ITMP1);
3356
3357                                 /* calculate interface checkcast code size */
3358
3359                                 s2 = 3; /* mov_membase_reg */
3360                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3361
3362                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3363                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3364                                         3 /* test */ + 6 /* jcc */;
3365
3366                                 if (!super)
3367                                         s2 += (opt_showdisassemble ? 5 : 0);
3368
3369                                 /* calculate class checkcast code size */
3370
3371                                 s3 = 3; /* mov_membase_reg */
3372                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3373                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3374
3375 #if 0
3376                                 if (s1 != REG_ITMP1) {
3377                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3378                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3379                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3380                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3381                                         a += 3;    /* sub */
3382
3383                                 } else
3384 #endif
3385                                         {
3386                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3387                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3388                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3389                                         }
3390
3391                                 s3 += 3 /* cmp */ + 6 /* jcc */;
3392
3393                                 if (!super)
3394                                         s3 += (opt_showdisassemble ? 5 : 0);
3395
3396                                 /* if class is not resolved, check which code to call */
3397
3398                                 if (!super) {
3399                                         M_TEST(s1);
3400                                         M_BEQ(6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3401
3402                                         codegen_addpatchref(cd, cd->mcodeptr,
3403                                                                                 PATCHER_checkcast_instanceof_flags,
3404                                                                                 (constant_classref *) iptr->target, 0);
3405
3406                                         if (opt_showdisassemble) {
3407                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3408                                         }
3409
3410                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
3411                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
3412                                         M_BEQ(s2 + 5);
3413                                 }
3414
3415                                 /* interface checkcast code */
3416
3417                                 if (!super || (super->flags & ACC_INTERFACE)) {
3418                                         if (super) {
3419                                                 M_TEST(s1);
3420                                                 M_BEQ(s2);
3421                                         }
3422
3423                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3424
3425                                         if (!super) {
3426                                                 codegen_addpatchref(cd, cd->mcodeptr,
3427                                                                                         PATCHER_checkcast_instanceof_interface,
3428                                                                                         (constant_classref *) iptr->target, 0);
3429
3430                                                 if (opt_showdisassemble) {
3431                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3432                                                 }
3433                                         }
3434
3435                                         x86_64_movl_membase32_reg(cd, REG_ITMP2,
3436                                                                                           OFFSET(vftbl_t, interfacetablelength),
3437                                                                                           REG_ITMP3);
3438                                         /* XXX TWISTI: should this be int arithmetic? */
3439                                         M_LSUB_IMM32(superindex, REG_ITMP3);
3440                                         M_TEST(REG_ITMP3);
3441                                         M_BLE(0);
3442                                         codegen_add_classcastexception_ref(cd, cd->mcodeptr);
3443                                         x86_64_mov_membase32_reg(cd, REG_ITMP2,
3444                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
3445                                                                                          superindex * sizeof(methodptr*),
3446                                                                                          REG_ITMP3);
3447                                         M_TEST(REG_ITMP3);
3448                                         M_BEQ(0);
3449                                         codegen_add_classcastexception_ref(cd, cd->mcodeptr);
3450
3451                                         if (!super)
3452                                                 M_JMP_IMM(s3);
3453                                 }
3454
3455                                 /* class checkcast code */
3456
3457                                 if (!super || !(super->flags & ACC_INTERFACE)) {
3458                                         if (super) {
3459                                                 M_TEST(s1);
3460                                                 M_BEQ(s3);
3461                                         }
3462
3463                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3464
3465                                         if (!super) {
3466                                                 codegen_addpatchref(cd, cd->mcodeptr,
3467                                                                                         PATCHER_checkcast_class,
3468                                                                                         (constant_classref *) iptr->target,
3469                                                                                         0);
3470
3471                                                 if (opt_showdisassemble) {
3472                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3473                                                 }
3474                                         }
3475
3476                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3477 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3478                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3479 #endif
3480                                         x86_64_movl_membase32_reg(cd, REG_ITMP2,
3481                                                                                           OFFSET(vftbl_t, baseval),
3482                                                                                           REG_ITMP2);
3483                                         /*                                      if (s1 != REG_ITMP1) { */
3484                                         /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3485                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
3486                                         /*                                                                                              REG_ITMP1); */
3487                                         /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3488                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3489                                         /*                                                                                              REG_ITMP3); */
3490                                         /*  #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
3491                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3492                                         /*  #endif */
3493                                         /*                                              x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP1, REG_ITMP2); */
3494
3495                                         /*                                      } else { */
3496                                         x86_64_movl_membase32_reg(cd, REG_ITMP3,
3497                                                                                           OFFSET(vftbl_t, baseval),
3498                                                                                           REG_ITMP3);
3499                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3500                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3501                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3502                                         /*                                      } */
3503 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3504                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3505 #endif
3506                                         M_CMP(REG_ITMP3, REG_ITMP2);
3507                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3508                                         codegen_add_classcastexception_ref(cd, cd->mcodeptr);
3509                                 }
3510                                 d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3511
3512                         } else {
3513                                 /* array type cast-check */
3514
3515                                 var_to_reg_int(s1, src, REG_ITMP1);
3516                                 M_INTMOVE(s1, rd->argintregs[0]);
3517
3518                                 if (iptr->val.a == NULL) {
3519                                         codegen_addpatchref(cd, cd->mcodeptr,
3520                                                                                 PATCHER_builtin_arraycheckcast,
3521                                                                                 (constant_classref *) iptr->target, 0);
3522
3523                                         if (opt_showdisassemble) {
3524                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3525                                         }
3526                                 }
3527
3528                                 M_MOV_IMM(iptr->val.a, rd->argintregs[1]);
3529                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
3530                                 M_CALL(REG_ITMP1);
3531                                 M_TEST(REG_RESULT);
3532                                 M_BEQ(0);
3533                                 codegen_add_classcastexception_ref(cd, cd->mcodeptr);
3534
3535                                 var_to_reg_int(s1, src, REG_ITMP1);
3536                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
3537                         }
3538                         M_INTMOVE(s1, d);
3539                         store_reg_to_var_int(iptr->dst, d);
3540                         break;
3541
3542                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3543
3544                                       /* op1:   0 == array, 1 == class                */
3545                                       /* val.a: (classinfo *) superclass              */
3546
3547                         /*  superclass is an interface:
3548                          *
3549                          *  return (sub != NULL) &&
3550                          *         (sub->vftbl->interfacetablelength > super->index) &&
3551                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3552                          *
3553                          *  superclass is a class:
3554                          *
3555                          *  return ((sub != NULL) && (0
3556                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3557                          *          super->vftbl->diffvall));
3558                          */
3559
3560                         {
3561                         classinfo *super;
3562                         vftbl_t   *supervftbl;
3563                         s4         superindex;
3564
3565                         super = (classinfo *) iptr->val.a;
3566
3567                         if (!super) {
3568                                 superindex = 0;
3569                                 supervftbl = NULL;
3570
3571                         } else {
3572                                 superindex = super->index;
3573                                 supervftbl = super->vftbl;
3574                         }
3575
3576 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3577             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3578 #endif
3579
3580                         var_to_reg_int(s1, src, REG_ITMP1);
3581                         d = reg_of_var(rd, iptr->dst, REG_ITMP2);
3582                         if (s1 == d) {
3583                                 M_INTMOVE(s1, REG_ITMP1);
3584                                 s1 = REG_ITMP1;
3585                         }
3586
3587                         /* calculate interface instanceof code size */
3588
3589                         s2 = 3; /* mov_membase_reg */
3590                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3591                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3592                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3593                                 3 /* test */ + 4 /* setcc */;
3594
3595                         if (!super)
3596                                 s2 += (opt_showdisassemble ? 5 : 0);
3597
3598                         /* calculate class instanceof code size */
3599
3600                         s3 = 3; /* mov_membase_reg */
3601                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3602                         s3 += 10; /* mov_imm_reg */
3603                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3604                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3605                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3606                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3607                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3608                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3609                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3610
3611                         if (!super)
3612                                 s3 += (opt_showdisassemble ? 5 : 0);
3613
3614                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3615
3616                         /* if class is not resolved, check which code to call */
3617
3618                         if (!super) {
3619                                 x86_64_test_reg_reg(cd, s1, s1);
3620                                 x86_64_jcc(cd, X86_64_CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3621                                                                                          7 + 6 + s2 + 5 + s3));
3622
3623                                 codegen_addpatchref(cd, cd->mcodeptr,
3624                                                                         PATCHER_checkcast_instanceof_flags,
3625                                                                         (constant_classref *) iptr->target, 0);
3626
3627                                 if (opt_showdisassemble) {
3628                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3629                                 }
3630
3631                                 x86_64_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3632                                 x86_64_alul_imm_reg(cd, X86_64_AND, ACC_INTERFACE, REG_ITMP3);
3633                                 x86_64_jcc(cd, X86_64_CC_Z, s2 + 5);
3634                         }
3635
3636                         /* interface instanceof code */
3637
3638                         if (!super || (super->flags & ACC_INTERFACE)) {
3639                                 if (super) {
3640                                         x86_64_test_reg_reg(cd, s1, s1);
3641                                         x86_64_jcc(cd, X86_64_CC_Z, s2);
3642                                 }
3643
3644                                 x86_64_mov_membase_reg(cd, s1,
3645                                                                            OFFSET(java_objectheader, vftbl),
3646                                                                            REG_ITMP1);
3647                                 if (!super) {
3648                                         codegen_addpatchref(cd, cd->mcodeptr,
3649                                                                                 PATCHER_checkcast_instanceof_interface,
3650                                                                                 (constant_classref *) iptr->target, 0);
3651
3652                                         if (opt_showdisassemble) {
3653                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3654                                         }
3655                                 }
3656
3657                                 x86_64_movl_membase32_reg(cd, REG_ITMP1,
3658                                                                                   OFFSET(vftbl_t, interfacetablelength),
3659                                                                                   REG_ITMP3);
3660                                 x86_64_alu_imm32_reg(cd, X86_64_SUB, superindex, REG_ITMP3);
3661                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3662
3663                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3664
3665                                 x86_64_jcc(cd, X86_64_CC_LE, a);
3666                                 x86_64_mov_membase32_reg(cd, REG_ITMP1,
3667                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3668                                                                                  superindex * sizeof(methodptr*),
3669                                                                                  REG_ITMP1);
3670                                 x86_64_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3671                                 x86_64_setcc_reg(cd, X86_64_CC_NE, d);
3672
3673                                 if (!super)
3674                                         x86_64_jmp_imm(cd, s3);
3675                         }
3676
3677                         /* class instanceof code */
3678
3679                         if (!super || !(super->flags & ACC_INTERFACE)) {
3680                                 if (super) {
3681                                         x86_64_test_reg_reg(cd, s1, s1);
3682                                         x86_64_jcc(cd, X86_64_CC_E, s3);
3683                                 }
3684
3685                                 x86_64_mov_membase_reg(cd, s1,
3686                                                                            OFFSET(java_objectheader, vftbl),
3687                                                                            REG_ITMP1);
3688
3689                                 if (!super) {
3690                                         codegen_addpatchref(cd, cd->mcodeptr,
3691                                                                                 PATCHER_instanceof_class,
3692                                                                                 (constant_classref *) iptr->target, 0);
3693
3694                                         if (opt_showdisassemble) {
3695                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3696                                         }
3697                                 }
3698
3699                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3700 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3701                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3702 #endif
3703                                 x86_64_movl_membase_reg(cd, REG_ITMP1,
3704                                                                                 OFFSET(vftbl_t, baseval),
3705                                                                                 REG_ITMP1);
3706                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3707                                                                                 OFFSET(vftbl_t, diffval),
3708                                                                                 REG_ITMP3);
3709                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3710                                                                                 OFFSET(vftbl_t, baseval),
3711                                                                                 REG_ITMP2);
3712 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3713                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3714 #endif
3715                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
3716                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d); /* may be REG_ITMP2 */
3717                                 x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP3, REG_ITMP1);
3718                                 x86_64_setcc_reg(cd, X86_64_CC_BE, d);
3719                         }
3720                         store_reg_to_var_int(iptr->dst, d);
3721                         }
3722                         break;
3723
3724                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3725                                          /* op1 = dimension, val.a = class            */
3726
3727                         /* check for negative sizes and copy sizes to stack if necessary  */
3728
3729                         MCODECHECK((10 * 4 * iptr->op1) + 5 + 10 * 8);
3730
3731                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3732                                 /* copy SAVEDVAR sizes to stack */
3733
3734                                 if (src->varkind != ARGVAR) {
3735                                         var_to_reg_int(s2, src, REG_ITMP1);
3736                                         M_LST(s2, REG_SP, s1 * 8);
3737                                 }
3738                         }
3739
3740                         /* is a patcher function set? */
3741
3742                         if (iptr->val.a == NULL) {
3743                                 codegen_addpatchref(cd, cd->mcodeptr,
3744                                                                         PATCHER_builtin_multianewarray,
3745                                                                         (constant_classref *) iptr->target, 0);
3746
3747                                 if (opt_showdisassemble) {
3748                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3749                                 }
3750
3751                                 a = 0;
3752
3753                         } else {
3754                                 a = (ptrint) iptr->val.a;
3755                         }
3756
3757                         /* a0 = dimension count */
3758
3759                         M_MOV_IMM(iptr->op1, rd->argintregs[0]);
3760
3761                         /* a1 = arrayvftbl */
3762
3763                         M_MOV_IMM(iptr->val.a, rd->argintregs[1]);
3764
3765                         /* a2 = pointer to dimensions = stack pointer */
3766
3767                         M_MOV(REG_SP, rd->argintregs[2]);
3768
3769                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3770                         M_CALL(REG_ITMP1);
3771
3772                         /* check for exception before result assignment */
3773
3774                         M_TEST(REG_RESULT);
3775                         M_BEQ(0);
3776                         codegen_add_fillinstacktrace_ref(cd, cd->mcodeptr);
3777
3778                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3779                         M_INTMOVE(REG_RESULT, s1);
3780                         store_reg_to_var_int(iptr->dst, s1);
3781                         break;
3782
3783                 default:
3784                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3785                         return false;
3786         } /* switch */
3787
3788         } /* for instruction */
3789
3790         /* copy values to interface registers */
3791
3792         src = bptr->outstack;
3793         len = bptr->outdepth;
3794         MCODECHECK(512);
3795 #if defined(ENABLE_LSRA)
3796         if (!opt_lsra)
3797 #endif
3798         while (src) {
3799                 len--;
3800                 if ((src->varkind != STACKVAR)) {
3801                         s2 = src->type;
3802                         if (IS_FLT_DBL_TYPE(s2)) {
3803                                 var_to_reg_flt(s1, src, REG_FTMP1);
3804                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3805                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3806
3807                                 } else {
3808                                         x86_64_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3809                                 }
3810
3811                         } else {
3812                                 var_to_reg_int(s1, src, REG_ITMP1);
3813                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3814                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3815
3816                                 } else {
3817                                         x86_64_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3818                                 }
3819                         }
3820                 }
3821                 src = src->prev;
3822         }
3823
3824         /* At the end of a basic block we may have to append some nops,
3825            because the patcher stub calling code might be longer than the
3826            actual instruction. So codepatching does not change the
3827            following block unintentionally. */
3828
3829         if (cd->mcodeptr < cd->lastmcodeptr) {
3830                 while (cd->mcodeptr < cd->lastmcodeptr) {
3831                         M_NOP;
3832                 }
3833         }
3834
3835         } /* if (bptr -> flags >= BBREACHED) */
3836         } /* for basic block */
3837
3838         dseg_createlinenumbertable(cd);
3839
3840
3841         /* generate exception and patcher stubs */
3842
3843         {
3844                 exceptionref *eref;
3845                 patchref     *pref;
3846                 ptrint        mcode;
3847                 u1           *savedmcodeptr;
3848                 u1           *tmpmcodeptr;
3849
3850                 savedmcodeptr = NULL;
3851
3852                 /* generate exception stubs */
3853
3854                 for (eref = cd->exceptionrefs; eref != NULL; eref = eref->next) {
3855                         gen_resolvebranch(cd->mcodebase + eref->branchpos,
3856                                                           eref->branchpos,
3857                                                           cd->mcodeptr - cd->mcodebase);
3858
3859                         MCODECHECK(512);
3860
3861                         /* Check if the exception is an
3862                            ArrayIndexOutOfBoundsException.  If so, move index register
3863                            into REG_ITMP1. */
3864
3865                         if (eref->reg != -1)
3866                                 M_MOV(eref->reg, REG_ITMP1);
3867
3868                         /* calcuate exception address */
3869
3870                         M_MOV_IMM(0, REG_ITMP2_XPC);
3871                         dseg_adddata(cd, cd->mcodeptr);
3872                         M_AADD_IMM32(eref->branchpos - 6, REG_ITMP2_XPC);
3873
3874                         /* move function to call into REG_ITMP3 */
3875
3876                         M_MOV_IMM(eref->function, REG_ITMP3);
3877
3878                         if (savedmcodeptr != NULL) {
3879                                 M_JMP_IMM(savedmcodeptr - cd->mcodeptr - 5);
3880
3881                         } else {
3882                                 savedmcodeptr = cd->mcodeptr;
3883
3884                                 x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3885                                 M_MOV(REG_SP, rd->argintregs[1]);
3886                                 M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3887                                 M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3888                                 M_MOV(REG_ITMP1, rd->argintregs[4]);            /* for AIOOBE */
3889
3890                                 M_ASUB_IMM(2 * 8, REG_SP);
3891                                 M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3892
3893                                 M_CALL(REG_ITMP3);
3894
3895                                 M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3896                                 M_AADD_IMM(2 * 8, REG_SP);
3897
3898                                 M_MOV_IMM(asm_handle_exception, REG_ITMP3);
3899                                 M_JMP(REG_ITMP3);
3900                         }
3901                 }
3902
3903
3904                 /* generate code patching stub call code */
3905
3906                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
3907                         /* check size of code segment */
3908
3909                         MCODECHECK(512);
3910
3911                         /* Get machine code which is patched back in later. A
3912                            `call rel32' is 5 bytes long (but read 8 bytes). */
3913
3914                         savedmcodeptr = cd->mcodebase + pref->branchpos;
3915                         mcode = *((ptrint *) savedmcodeptr);
3916
3917                         /* patch in `call rel32' to call the following code */
3918
3919                         tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr          */
3920                         cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position */
3921
3922                         M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
3923
3924                         cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr   */
3925
3926                         /* move pointer to java_objectheader onto stack */
3927
3928 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3929                         /* create a virtual java_objectheader */
3930
3931                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
3932                         a = dseg_addaddress(cd, NULL);                      /* vftbl      */
3933
3934                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP3);
3935                         M_PUSH(REG_ITMP3);
3936 #else
3937                         M_PUSH_IMM(0);
3938 #endif
3939
3940                         /* move machine code bytes and classinfo pointer into registers */
3941
3942                         M_MOV_IMM(mcode, REG_ITMP3);
3943                         M_PUSH(REG_ITMP3);
3944                         M_MOV_IMM(pref->ref, REG_ITMP3);
3945                         M_PUSH(REG_ITMP3);
3946                         M_MOV_IMM(pref->disp, REG_ITMP3);
3947                         M_PUSH(REG_ITMP3);
3948
3949                         M_MOV_IMM(pref->patcher, REG_ITMP3);
3950                         M_PUSH(REG_ITMP3);
3951
3952                         M_MOV_IMM(asm_wrapper_patcher, REG_ITMP3);
3953                         M_JMP(REG_ITMP3);
3954                 }
3955         }
3956
3957         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
3958
3959         /* everything's ok */
3960
3961         return true;
3962 }
3963
3964
3965 /* createcompilerstub **********************************************************
3966
3967    Creates a stub routine which calls the compiler.
3968
3969 *******************************************************************************/
3970
3971 #define COMPILERSTUB_DATASIZE    2 * SIZEOF_VOID_P
3972 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
3973
3974 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
3975
3976
3977 u1 *createcompilerstub(methodinfo *m)
3978 {
3979         u1          *s;                     /* memory to hold the stub            */
3980         ptrint      *d;
3981         codegendata *cd;
3982         s4           dumpsize;
3983
3984         s = CNEW(u1, COMPILERSTUB_SIZE);
3985
3986         /* set data pointer and code pointer */
3987
3988         d = (ptrint *) s;
3989         s = s + COMPILERSTUB_DATASIZE;
3990
3991         /* mark start of dump memory area */
3992
3993         dumpsize = dump_size();
3994
3995         cd = DNEW(codegendata);
3996         cd->mcodeptr = s;
3997
3998         /* Store the methodinfo* in the same place as in the methodheader
3999            for compiled methods. */
4000
4001         d[0] = (ptrint) asm_call_jit_compiler;
4002         d[1] = (ptrint) m;
4003
4004         /* code for the stub */
4005
4006         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 1 * SIZEOF_VOID_P));       /* methodinfo  */
4007         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 2 * SIZEOF_VOID_P));  /* compiler pointer */
4008         M_JMP(REG_ITMP3);
4009
4010 #if defined(ENABLE_STATISTICS)
4011         if (opt_stat)
4012                 count_cstub_len += COMPILERSTUB_SIZE;
4013 #endif
4014
4015         /* release dump area */
4016
4017         dump_release(dumpsize);
4018
4019         return s;
4020 }
4021
4022
4023 /* createnativestub ************************************************************
4024
4025    Creates a stub routine which calls a native method.
4026
4027 *******************************************************************************/
4028
4029 u1 *createnativestub(functionptr f, methodinfo *m, codegendata *cd,
4030                                          registerdata *rd, methoddesc *nmd)
4031 {
4032         methoddesc *md;
4033         s4          stackframesize;         /* size of stackframe if needed       */
4034         s4          nativeparams;
4035         s4          i, j;                   /* count variables                    */
4036         s4          t;
4037         s4          s1, s2;
4038
4039         /* initialize variables */
4040
4041         md = m->parseddesc;
4042         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
4043
4044         /* calculate stack frame size */
4045
4046         stackframesize =
4047                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
4048                 sizeof(localref_table) / SIZEOF_VOID_P +
4049                 INT_ARG_CNT + FLT_ARG_CNT + 1 +         /* + 1 for function address   */
4050                 nmd->memuse;
4051
4052         if (!(stackframesize & 0x1))                /* keep stack 16-byte aligned */
4053                 stackframesize++;
4054
4055         /* create method header */
4056
4057         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
4058         (void) dseg_adds4(cd, stackframesize * 8);              /* FrameSize      */
4059         (void) dseg_adds4(cd, 0);                               /* IsSync         */
4060         (void) dseg_adds4(cd, 0);                               /* IsLeaf         */
4061         (void) dseg_adds4(cd, 0);                               /* IntSave        */
4062         (void) dseg_adds4(cd, 0);                               /* FltSave        */
4063         (void) dseg_addlinenumbertablesize(cd);
4064         (void) dseg_adds4(cd, 0);                               /* ExTableSize    */
4065
4066         /* initialize mcode variables */
4067
4068         cd->mcodeptr = (u1 *) cd->mcodebase;
4069         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
4070
4071         /* generate native method profiling code */
4072
4073         if (opt_prof) {
4074                 /* count frequency */
4075
4076                 M_MOV_IMM(m, REG_ITMP2);
4077                 M_IINC_MEMBASE(REG_ITMP2, OFFSET(methodinfo, frequency));
4078         }
4079
4080         /* generate stub code */
4081
4082         M_ASUB_IMM(stackframesize * 8, REG_SP);
4083
4084         if (opt_verbosecall) {
4085                 /* save integer and float argument registers */
4086
4087                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4088                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4089                                 M_LST(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4090
4091                 for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4092                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4093                                 M_DST(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4094
4095                 /* show integer hex code for float arguments */
4096
4097                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++) {
4098                         /* if the paramtype is a float, we have to right shift all
4099                            following integer registers */
4100
4101                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type)) {
4102                                 for (s1 = INT_ARG_CNT - 2; s1 >= i; s1--)
4103                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
4104
4105                                 x86_64_movd_freg_reg(cd, rd->argfltregs[j], rd->argintregs[i]);
4106                                 j++;
4107                         }
4108                 }
4109
4110                 M_MOV_IMM(m, REG_ITMP1);
4111                 M_AST(REG_ITMP1, REG_SP, 0 * 8);
4112                 M_MOV_IMM(builtin_trace_args, REG_ITMP1);
4113                 M_CALL(REG_ITMP1);
4114
4115                 /* restore integer and float argument registers */
4116
4117                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4118                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4119                                 M_LLD(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4120
4121                 for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4122                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4123                                 M_DLD(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4124         }
4125
4126
4127         /* get function address (this must happen before the stackframeinfo) */
4128
4129 #if !defined(WITH_STATIC_CLASSPATH)
4130         if (f == NULL) {
4131                 codegen_addpatchref(cd, cd->mcodeptr, PATCHER_resolve_native, m, 0);
4132
4133                 if (opt_showdisassemble) {
4134                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
4135                 }
4136         }
4137 #endif
4138
4139         M_MOV_IMM(f, REG_ITMP3);
4140
4141
4142         /* save integer and float argument registers */
4143
4144         for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4145                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4146                         M_LST(rd->argintregs[j++], REG_SP, i * 8);
4147
4148         for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4149                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4150                         M_DST(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4151
4152         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4153
4154         /* create dynamic stack info */
4155
4156         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4157         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[1]);
4158         M_ALEA(REG_SP, stackframesize * 8 + SIZEOF_VOID_P, rd->argintregs[2]);
4159         M_ALD(rd->argintregs[3], REG_SP, stackframesize * 8);
4160         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
4161         M_CALL(REG_ITMP1);
4162
4163         /* restore integer and float argument registers */
4164
4165         for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4166                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4167                         M_LLD(rd->argintregs[j++], REG_SP, i * 8);
4168
4169         for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4170                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4171                         M_DLD(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4172
4173         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4174
4175
4176         /* copy or spill arguments to new locations */
4177
4178         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
4179                 t = md->paramtypes[i].type;
4180
4181                 if (IS_INT_LNG_TYPE(t)) {
4182                         if (!md->params[i].inmemory) {
4183                                 s1 = rd->argintregs[md->params[i].regoff];
4184
4185                                 if (!nmd->params[j].inmemory) {
4186                                         s2 = rd->argintregs[nmd->params[j].regoff];
4187                                         M_INTMOVE(s1, s2);
4188
4189                                 } else {
4190                                         s2 = nmd->params[j].regoff;
4191                                         M_LST(s1, REG_SP, s2 * 8);
4192                                 }
4193
4194                         } else {
4195                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4196                                 s2 = nmd->params[j].regoff;
4197                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
4198                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
4199                         }
4200
4201                 } else {
4202                         /* We only copy spilled float arguments, as the float argument    */
4203                         /* registers keep unchanged.                                      */
4204
4205                         if (md->params[i].inmemory) {
4206                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4207                                 s2 = nmd->params[j].regoff;
4208                                 M_DLD(REG_FTMP1, REG_SP, s1 * 8);
4209                                 M_DST(REG_FTMP1, REG_SP, s2 * 8);
4210                         }
4211                 }
4212         }
4213
4214         /* put class into second argument register */
4215
4216         if (m->flags & ACC_STATIC)
4217                 M_MOV_IMM(m->class, rd->argintregs[1]);
4218
4219         /* put env into first argument register */
4220
4221         M_MOV_IMM(_Jv_env, rd->argintregs[0]);
4222
4223         /* do the native function call */
4224
4225         M_CALL(REG_ITMP3);
4226
4227         /* save return value */
4228
4229         if (md->returntype.type != TYPE_VOID) {
4230                 if (IS_INT_LNG_TYPE(md->returntype.type))
4231                         M_LST(REG_RESULT, REG_SP, 0 * 8);
4232                 else
4233                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
4234         }
4235
4236         /* remove native stackframe info */
4237
4238         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4239         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
4240         M_CALL(REG_ITMP1);
4241
4242         /* generate call trace */
4243
4244         if (opt_verbosecall) {
4245                 /* just restore the value we need, don't care about the other */
4246
4247                 if (md->returntype.type != TYPE_VOID) {
4248                         if (IS_INT_LNG_TYPE(md->returntype.type))
4249                                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
4250                         else
4251                                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4252                 }
4253
4254                 M_MOV_IMM(m, rd->argintregs[0]);
4255                 M_MOV(REG_RESULT, rd->argintregs[1]);
4256                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4257                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4258
4259                 M_MOV_IMM(builtin_displaymethodstop, REG_ITMP1);
4260                 M_CALL(REG_ITMP1);
4261         }
4262
4263         /* check for exception */
4264
4265 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4266         M_MOV_IMM(builtin_get_exceptionptrptr, REG_ITMP3);
4267         M_CALL(REG_ITMP3);
4268 #else
4269         M_MOV_IMM(&_no_threads_exceptionptr, REG_RESULT);
4270 #endif
4271         M_ALD(REG_ITMP2, REG_RESULT, 0);
4272
4273         /* restore return value */
4274
4275         if (md->returntype.type != TYPE_VOID) {
4276                 if (IS_INT_LNG_TYPE(md->returntype.type))
4277                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
4278                 else
4279                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4280         }
4281
4282         /* test for exception */
4283
4284         M_TEST(REG_ITMP2);
4285         M_BNE(7 + 1);
4286
4287         /* remove stackframe */
4288
4289         M_AADD_IMM(stackframesize * 8, REG_SP);
4290         M_RET;
4291
4292
4293         /* handle exception */
4294
4295 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4296         M_LST(REG_ITMP2, REG_SP, 0 * 8);
4297         M_MOV_IMM(builtin_get_exceptionptrptr, REG_ITMP3);
4298         M_CALL(REG_ITMP3);
4299         M_AST_IMM32(0, REG_RESULT, 0);                 /* clear exception pointer */
4300         M_LLD(REG_ITMP1_XPTR, REG_SP, 0 * 8);
4301 #else
4302         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
4303         M_MOV_IMM(&_no_threads_exceptionptr, REG_ITMP3);
4304         M_AST_IMM32(0, REG_ITMP3, 0);                  /* clear exception pointer */
4305 #endif
4306
4307         /* remove stackframe */
4308
4309         M_AADD_IMM(stackframesize * 8, REG_SP);
4310
4311         M_LLD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
4312         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
4313
4314         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
4315         M_JMP(REG_ITMP3);
4316
4317
4318         /* process patcher calls **************************************************/
4319
4320         {
4321                 patchref *pref;
4322                 ptrint    mcode;
4323                 u1       *savedmcodeptr;
4324                 u1       *tmpmcodeptr;
4325 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4326                 s4        disp;
4327 #endif
4328
4329                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4330                         /* Get machine code which is patched back in later. A
4331                            `call rel32' is 5 bytes long (but read 8 bytes). */
4332
4333                         savedmcodeptr = cd->mcodebase + pref->branchpos;
4334                         mcode = *((ptrint *) savedmcodeptr);
4335
4336                         /* patch in `call rel32' to call the following code               */
4337
4338                         tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr          */
4339                         cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position */
4340
4341                         M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
4342
4343                         cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr   */
4344
4345                         /* move pointer to java_objectheader onto stack */
4346
4347 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4348                         /* create a virtual java_objectheader */
4349
4350                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4351                         disp = dseg_addaddress(cd, NULL);                   /* vftbl      */
4352
4353                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP3);
4354                         M_PUSH(REG_ITMP3);
4355 #else
4356                         M_PUSH_IMM(0);
4357 #endif
4358
4359                         /* move machine code bytes and classinfo pointer into registers */
4360
4361                         M_MOV_IMM(mcode, REG_ITMP3);
4362                         M_PUSH(REG_ITMP3);
4363                         M_MOV_IMM(pref->ref, REG_ITMP3);
4364                         M_PUSH(REG_ITMP3);
4365                         M_MOV_IMM(pref->disp, REG_ITMP3);
4366                         M_PUSH(REG_ITMP3);
4367
4368                         M_MOV_IMM(pref->patcher, REG_ITMP3);
4369                         M_PUSH(REG_ITMP3);
4370
4371                         M_MOV_IMM(asm_wrapper_patcher, REG_ITMP3);
4372                         M_JMP(REG_ITMP3);
4373                 }
4374         }
4375
4376         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
4377
4378         return cd->code->entrypoint;
4379 }
4380
4381
4382 /*
4383  * These are local overrides for various environment variables in Emacs.
4384  * Please do not remove this and leave it at the end of the file, where
4385  * Emacs will automagically detect them.
4386  * ---------------------------------------------------------------------
4387  * Local variables:
4388  * mode: c
4389  * indent-tabs-mode: t
4390  * c-basic-offset: 4
4391  * tab-width: 4
4392  * End:
4393  * vim:noexpandtab:sw=4:ts=4:
4394  */