src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31             Edwin Steiner
  32
  33    $Id: codegen.c 4764 2006-04-13 09:32:09Z twisti $
  34
  35 */
  36
  37
  38 #include "config.h"
  39
  40 #include <assert.h>
  41 #include <stdio.h>
  42
  43 #include "vm/types.h"
  44
  45 #include "md.h"
  46 #include "md-abi.h"
  47
  48 #include "vm/jit/x86_64/arch.h"
  49 #include "vm/jit/x86_64/codegen.h"
  50 #include "vm/jit/x86_64/emitfuncs.h"
  51
  52 #include "native/native.h"
  53 #include "vm/builtin.h"
  54 #include "vm/exceptions.h"
  55 #include "vm/global.h"
  56 #include "vm/loader.h"
  57 #include "vm/options.h"
  58 #include "vm/statistics.h"
  59 #include "vm/stringlocal.h"
  60 #include "vm/vm.h"
  61 #include "vm/jit/asmpart.h"
  62 #include "vm/jit/codegen-common.h"
  63 #include "vm/jit/dseg.h"
  64 #include "vm/jit/jit.h"
  65 #include "vm/jit/methodheader.h"
  66 #include "vm/jit/parse.h"
  67 #include "vm/jit/patcher.h"
  68 #include "vm/jit/reg.h"
  69 #include "vm/jit/replace.h"
  70
  71 #if defined(ENABLE_LSRA)
  72 # include "vm/jit/allocator/lsra.h"
  73 #endif
  74
  75
  76 /* codegen *********************************************************************
  77
  78    Generates machine code.
  79
  80 *******************************************************************************/
  81
  82 bool codegen(jitdata *jd)
  83 {
  84         methodinfo         *m;
  85         codegendata        *cd;
  86         registerdata       *rd;
  87         s4                  len, s1, s2, s3, d, disp;
  88         u2                  currentline;
  89         ptrint              a;
  90         s4                  stackframesize;
  91         stackptr            src;
  92         varinfo            *var;
  93         basicblock         *bptr;
  94         instruction        *iptr;
  95         exceptiontable     *ex;
  96         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
  97         builtintable_entry *bte;
  98         methoddesc         *md;
  99         rplpoint           *replacementpoint;
 100
 101         /* get required compiler data */
 102
 103         m  = jd->m;
 104         cd = jd->cd;
 105         rd = jd->rd;
 106
 107         /* prevent compiler warnings */
 108
 109         d = 0;
 110         lm = NULL;
 111         bte = NULL;
 112
 113         {
 114         s4 i, p, t, l;
 115         s4 savedregs_num;
 116
 117         savedregs_num = 0;
 118
 119         /* space to save used callee saved registers */
 120
 121         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 122         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 123
 124         stackframesize = rd->memuse + savedregs_num;
 125
 126 #if defined(USE_THREADS)
 127         /* space to save argument of monitor_enter */
 128
 129         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 130                 stackframesize++;
 131 #endif
 132
 133     /* Keep stack of non-leaf functions 16-byte aligned for calls into native */
 134         /* code e.g. libc or jni (alignment problems with movaps).                */
 135
 136         if (!m->isleafmethod || opt_verbosecall)
 137                 stackframesize |= 0x1;
 138
 139         /* create method header */
 140
 141         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 142         (void) dseg_adds4(cd, stackframesize * 8);              /* FrameSize      */
 143
 144 #if defined(USE_THREADS)
 145         /* IsSync contains the offset relative to the stack pointer for the
 146            argument of monitor_exit used in the exception handler. Since the
 147            offset could be zero and give a wrong meaning of the flag it is
 148            offset by one.
 149         */
 150
 151         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 152                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);        /* IsSync         */
 153         else
 154 #endif
 155                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 156
 157         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 158         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse);  /* IntSave        */
 159         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse);  /* FltSave        */
 160
 161         (void) dseg_addlinenumbertablesize(cd);
 162
 163         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 164
 165         /* create exception table */
 166
 167         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 168                 dseg_addtarget(cd, ex->start);
 169                 dseg_addtarget(cd, ex->end);
 170                 dseg_addtarget(cd, ex->handler);
 171                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 172         }
 173
 174         /* initialize mcode variables */
 175
 176         cd->mcodeptr = (u1 *) cd->mcodebase;
 177         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
 178
 179         /* initialize the last patcher pointer */
 180
 181         cd->lastmcodeptr = cd->mcodeptr;
 182
 183         /* generate method profiling code */
 184
 185         if (opt_prof) {
 186                 /* count frequency */
 187
 188                 M_MOV_IMM(m, REG_ITMP3);
 189                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(methodinfo, frequency));
 190
 191                 PROFILE_CYCLE_START;
 192         }
 193
 194         /* create stack frame (if necessary) */
 195
 196         if (stackframesize)
 197                 M_ASUB_IMM(stackframesize * 8, REG_SP);
 198
 199         /* save used callee saved registers */
 200
 201         p = stackframesize;
 202         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 203                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 204         }
 205         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 206                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 207         }
 208
 209         /* take arguments out of register or stack frame */
 210
 211         md = m->parseddesc;
 212
 213         for (p = 0, l = 0; p < md->paramcount; p++) {
 214                 t = md->paramtypes[p].type;
 215                 var = &(rd->locals[l][t]);
 216                 l++;
 217                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 218                         l++;
 219                 if (var->type < 0)
 220                         continue;
 221                 s1 = md->params[p].regoff;
 222                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 223                         s2 = rd->argintregs[s1];
 224                         if (!md->params[p].inmemory) {           /* register arguments    */
 225                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 226                                         M_INTMOVE(s2, var->regoff);
 227
 228                                 } else {                             /* reg arg -> spilled    */
 229                                     M_LST(s2, REG_SP, var->regoff * 8);
 230                                 }
 231
 232                         } else {                                 /* stack arguments       */
 233                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 234                                         /* + 8 for return address */
 235                                         M_LLD(var->regoff, REG_SP, (stackframesize + s1) * 8 + 8);
 236
 237                                 } else {                             /* stack arg -> spilled  */
 238                                         var->regoff = stackframesize + s1 + 1;
 239                                 }
 240                         }
 241
 242                 } else {                                     /* floating args         */
 243                         if (!md->params[p].inmemory) {           /* register arguments    */
 244                                 s2 = rd->argfltregs[s1];
 245                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 246                                         M_FLTMOVE(s2, var->regoff);
 247
 248                                 } else {                                         /* reg arg -> spilled    */
 249                                         M_DST(s2, REG_SP, var->regoff * 8);
 250                                 }
 251
 252                         } else {                                 /* stack arguments       */
 253                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 254                                         M_DLD(var->regoff, REG_SP, (stackframesize + s1) * 8 + 8);
 255
 256                                 } else {
 257                                         var->regoff = stackframesize + s1 + 1;
 258                                 }
 259                         }
 260                 }
 261         }  /* end for */
 262
 263         /* save monitorenter argument */
 264
 265 #if defined(USE_THREADS)
 266         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 267                 /* stack offset for monitor argument */
 268
 269                 s1 = rd->memuse;
 270
 271                 if (opt_verbosecall) {
 272                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 273
 274                         for (p = 0; p < INT_ARG_CNT; p++)
 275                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 276
 277                         for (p = 0; p < FLT_ARG_CNT; p++)
 278                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 279
 280                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 281                 }
 282
 283                 /* decide which monitor enter function to call */
 284
 285                 if (m->flags & ACC_STATIC) {
 286                         M_MOV_IMM(m->class, REG_ITMP1);
 287                         M_AST(REG_ITMP1, REG_SP, s1 * 8);
 288                         M_INTMOVE(REG_ITMP1, rd->argintregs[0]);
 289                         M_MOV_IMM(BUILTIN_staticmonitorenter, REG_ITMP1);
 290                         M_CALL(REG_ITMP1);
 291
 292                 } else {
 293                         M_TEST(rd->argintregs[0]);
 294                         M_BEQ(0);
 295                         codegen_add_nullpointerexception_ref(cd, cd->mcodeptr);
 296                         M_AST(rd->argintregs[0], REG_SP, s1 * 8);
 297                         M_MOV_IMM(BUILTIN_monitorenter, REG_ITMP1);
 298                         M_CALL(REG_ITMP1);
 299                 }
 300
 301                 if (opt_verbosecall) {
 302                         for (p = 0; p < INT_ARG_CNT; p++)
 303                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 304
 305                         for (p = 0; p < FLT_ARG_CNT; p++)
 306                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 307
 308                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 309                 }
 310         }
 311 #endif
 312
 313 #if !defined(NDEBUG)
 314         /* Copy argument registers to stack and call trace function with
 315            pointer to arguments on stack. */
 316
 317         if (opt_verbosecall) {
 318                 M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 319
 320                 /* save integer argument registers */
 321
 322                 for (p = 0; p < INT_ARG_CNT; p++)
 323                         M_LST(rd->argintregs[p], REG_SP, (1 + p) * 8);
 324
 325                 /* save float argument registers */
 326
 327                 for (p = 0; p < FLT_ARG_CNT; p++)
 328                         M_DST(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 329
 330                 /* save temporary registers for leaf methods */
 331
 332                 if (m->isleafmethod) {
 333                         for (p = 0; p < INT_TMP_CNT; p++)
 334                                 M_LST(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 335
 336                         for (p = 0; p < FLT_TMP_CNT; p++)
 337                                 M_DST(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 338                 }
 339
 340                 /* show integer hex code for float arguments */
 341
 342                 for (p = 0, l = 0; p < md->paramcount && p < INT_ARG_CNT; p++) {
 343                         /* If the paramtype is a float, we have to right shift all
 344                            following integer registers. */
 345
 346                         if (IS_FLT_DBL_TYPE(md->paramtypes[p].type)) {
 347                                 for (s1 = INT_ARG_CNT - 2; s1 >= p; s1--) {
 348                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
 349                                 }
 350
 351                                 x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 352                                 l++;
 353                         }
 354                 }
 355
 356                 M_MOV_IMM(m, REG_ITMP2);
 357                 M_AST(REG_ITMP2, REG_SP, 0 * 8);
 358                 M_MOV_IMM(builtin_trace_args, REG_ITMP1);
 359                 M_CALL(REG_ITMP1);
 360
 361                 /* restore integer argument registers */
 362
 363                 for (p = 0; p < INT_ARG_CNT; p++)
 364                         M_LLD(rd->argintregs[p], REG_SP, (1 + p) * 8);
 365
 366                 /* restore float argument registers */
 367
 368                 for (p = 0; p < FLT_ARG_CNT; p++)
 369                         M_DLD(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 370
 371                 /* restore temporary registers for leaf methods */
 372
 373                 if (m->isleafmethod) {
 374                         for (p = 0; p < INT_TMP_CNT; p++)
 375                                 M_LLD(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 376
 377                         for (p = 0; p < FLT_TMP_CNT; p++)
 378                                 M_DLD(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 379                 }
 380
 381                 M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 382         }
 383 #endif /* !defined(NDEBUG) */
 384
 385         }
 386
 387         /* end of header generation */
 388
 389         replacementpoint = jd->code->rplpoints;
 390
 391         /* walk through all basic blocks */
 392
 393         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 394
 395                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 396
 397                 if (bptr->flags >= BBREACHED) {
 398
 399                         /* branch resolving */
 400
 401                         branchref *bref;
 402                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 403                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 404                                                                   bref->branchpos,
 405                                                                   bptr->mpc);
 406                         }
 407
 408                 /* handle replacement points */
 409
 410                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 411                         replacementpoint->pc = (u1*)(ptrint)bptr->mpc; /* will be resolved later */
 412
 413                         replacementpoint++;
 414
 415                         assert(cd->lastmcodeptr <= cd->mcodeptr);
 416                         cd->lastmcodeptr = cd->mcodeptr + 5; /* 5 byte jmp patch */
 417                 }
 418
 419                 /* copy interface registers to their destination */
 420
 421                 src = bptr->instack;
 422                 len = bptr->indepth;
 423                 MCODECHECK(512);
 424
 425                 /* generate basicblock profiling code */
 426
 427                 if (opt_prof_bb) {
 428                         /* count frequency */
 429
 430                         M_MOV_IMM(m->bbfrequency, REG_ITMP2);
 431                         M_IINC_MEMBASE(REG_ITMP2, bptr->debug_nr * 4);
 432
 433                         /* if this is an exception handler, start profiling again */
 434
 435                         if (bptr->type == BBTYPE_EXH)
 436                                 PROFILE_CYCLE_START;
 437                 }
 438
 439 #if defined(ENABLE_LSRA)
 440                 if (opt_lsra) {
 441                         while (src != NULL) {
 442                                 len--;
 443                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 444                                         if (bptr->type == BBTYPE_SBR) {
 445                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 446                                                 if (!(src->flags & INMEMORY))
 447                                                         d= src->regoff;
 448                                                 else
 449                                                         d=REG_ITMP1;
 450                                                 x86_64_pop_reg(cd, d);
 451                                                 store_reg_to_var_int(src, d);
 452
 453                                         } else if (bptr->type == BBTYPE_EXH) {
 454                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 455                                                 if (!(src->flags & INMEMORY))
 456                                                         d= src->regoff;
 457                                                 else
 458                                                         d=REG_ITMP1;
 459                                                 M_INTMOVE(REG_ITMP1, d);
 460                                                 store_reg_to_var_int(src, d);
 461                                         }
 462                                 }
 463                                 src = src->prev;
 464                         }
 465
 466                 } else {
 467 #endif
 468
 469                 while (src != NULL) {
 470                         len--;
 471                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 472                                 if (bptr->type == BBTYPE_SBR) {
 473                                         d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 474                                         M_POP(d);
 475                                         store_reg_to_var_int(src, d);
 476
 477                                 } else if (bptr->type == BBTYPE_EXH) {
 478                                         d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 479                                         M_INTMOVE(REG_ITMP1, d);
 480                                         store_reg_to_var_int(src, d);
 481                                 }
 482
 483                         } else {
 484                                 d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 485                                 if ((src->varkind != STACKVAR)) {
 486                                         s2 = src->type;
 487                                         if (IS_FLT_DBL_TYPE(s2)) {
 488                                                 s1 = rd->interfaces[len][s2].regoff;
 489
 490                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 491                                                         M_FLTMOVE(s1, d);
 492                                                 else
 493                                                         M_DLD(d, REG_SP, s1 * 8);
 494
 495                                                 store_reg_to_var_flt(src, d);
 496
 497                                         } else {
 498                                                 s1 = rd->interfaces[len][s2].regoff;
 499
 500                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 501                                                         M_INTMOVE(s1, d);
 502                                                 else
 503                                                         M_LLD(d, REG_SP, s1 * 8);
 504
 505                                                 store_reg_to_var_int(src, d);
 506                                         }
 507                                 }
 508                         }
 509                         src = src->prev;
 510                 }
 511 #if defined(ENABLE_LSRA)
 512                 }
 513 #endif
 514                 /* walk through all instructions */
 515
 516                 src = bptr->instack;
 517                 len = bptr->icount;
 518                 currentline = 0;
 519
 520                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 521                         if (iptr->line != currentline) {
 522                                 dseg_addlinenumber(cd, iptr->line, cd->mcodeptr);
 523                                 currentline = iptr->line;
 524                         }
 525
 526                         MCODECHECK(1024);                         /* 1KB should be enough */
 527
 528                         switch (iptr->opc) {
 529                         case ICMD_INLINE_START: /* internal ICMDs                         */
 530                         case ICMD_INLINE_END:
 531                                 break;
 532
 533                         case ICMD_NOP:    /* ...  ==> ...                                 */
 534                                 break;
 535
 536                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 537
 538                                 if (src->flags & INMEMORY)
 539                                         M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
 540                                 else
 541                                         M_TEST(src->regoff);
 542                                 M_BEQ(0);
 543                                 codegen_add_nullpointerexception_ref(cd, cd->mcodeptr);
 544                                 break;
 545
 546                 /* constant operations ************************************************/
 547
 548                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 549                                       /* op1 = 0, val.i = constant                    */
 550
 551                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 552                         if (iptr->val.i == 0)
 553                                 M_CLR(d);
 554                         else
 555                                 M_IMOV_IMM(iptr->val.i, d);
 556                         store_reg_to_var_int(iptr->dst, d);
 557                         break;
 558
 559                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 560                                       /* op1 = 0, val.l = constant                    */
 561
 562                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 563                         if (iptr->val.l == 0)
 564                                 M_CLR(d);
 565                         else
 566                                 M_MOV_IMM(iptr->val.l, d);
 567                         store_reg_to_var_int(iptr->dst, d);
 568                         break;
 569
 570                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 571                                       /* op1 = 0, val.f = constant                    */
 572
 573                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 574                         disp = dseg_addfloat(cd, iptr->val.f);
 575                         x86_64_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + disp, d);
 576                         store_reg_to_var_flt(iptr->dst, d);
 577                         break;
 578
 579                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 580                                       /* op1 = 0, val.d = constant                    */
 581
 582                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 583                         disp = dseg_adddouble(cd, iptr->val.d);
 584                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, d);
 585                         store_reg_to_var_flt(iptr->dst, d);
 586                         break;
 587
 588                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 589                                       /* op1 = 0, val.a = constant                    */
 590
 591                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 592
 593                         if ((iptr->target != NULL) && (iptr->val.a == NULL)) {
 594 /*                              PROFILE_CYCLE_STOP; */
 595
 596                                 codegen_addpatchref(cd, cd->mcodeptr,
 597                                                                         PATCHER_aconst,
 598                                                                         (unresolved_class *) iptr->target, 0);
 599
 600                                 if (opt_showdisassemble) {
 601                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
 602                                 }
 603
 604 /*                              PROFILE_CYCLE_START; */
 605
 606                                 M_MOV_IMM(iptr->val.a, d);
 607
 608                         } else {
 609                                 if (iptr->val.a == 0)
 610                                         M_CLR(d);
 611                                 else
 612                                         M_MOV_IMM(iptr->val.a, d);
 613                         }
 614                         store_reg_to_var_int(iptr->dst, d);
 615                         break;
 616
 617
 618                 /* load/store operations **********************************************/
 619
 620                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 621                                       /* op1 = local variable                         */
 622
 623                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 624                         if ((iptr->dst->varkind == LOCALVAR) &&
 625                             (iptr->dst->varnum == iptr->op1)) {
 626                                 break;
 627                         }
 628                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 629                         if (var->flags & INMEMORY) {
 630                                 x86_64_movl_membase_reg(cd, REG_SP, var->regoff * 8, d);
 631                                 store_reg_to_var_int(iptr->dst, d);
 632
 633                         } else {
 634                                 if (iptr->dst->flags & INMEMORY) {
 635                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 636
 637                                 } else {
 638                                         M_INTMOVE(var->regoff, d);
 639                                 }
 640                         }
 641                         break;
 642
 643                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 644                 case ICMD_ALOAD:      /* op1 = local variable                         */
 645
 646                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 647                         if ((iptr->dst->varkind == LOCALVAR) &&
 648                             (iptr->dst->varnum == iptr->op1)) {
 649                                 break;
 650                         }
 651                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 652                         if (var->flags & INMEMORY) {
 653                                 x86_64_mov_membase_reg(cd, REG_SP, var->regoff * 8, d);
 654                                 store_reg_to_var_int(iptr->dst, d);
 655
 656                         } else {
 657                                 if (iptr->dst->flags & INMEMORY) {
 658                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 659
 660                                 } else {
 661                                         M_INTMOVE(var->regoff, d);
 662                                 }
 663                         }
 664                         break;
 665
 666                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 667                 case ICMD_DLOAD:      /* op1 = local variable                         */
 668
 669                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 670                         if ((iptr->dst->varkind == LOCALVAR) &&
 671                             (iptr->dst->varnum == iptr->op1)) {
 672                                 break;
 673                         }
 674                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 675                         if (var->flags & INMEMORY) {
 676                                 x86_64_movq_membase_reg(cd, REG_SP, var->regoff * 8, d);
 677                                 store_reg_to_var_flt(iptr->dst, d);
 678
 679                         } else {
 680                                 if (iptr->dst->flags & INMEMORY) {
 681                                         x86_64_movq_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 682
 683                                 } else {
 684                                         M_FLTMOVE(var->regoff, d);
 685                                 }
 686                         }
 687                         break;
 688
 689                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 690                 case ICMD_LSTORE:     /* op1 = local variable                         */
 691                 case ICMD_ASTORE:
 692
 693                         if ((src->varkind == LOCALVAR) &&
 694                             (src->varnum == iptr->op1)) {
 695                                 break;
 696                         }
 697                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 698                         if (var->flags & INMEMORY) {
 699                                 var_to_reg_int(s1, src, REG_ITMP1);
 700                                 x86_64_mov_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 701
 702                         } else {
 703                                 var_to_reg_int(s1, src, var->regoff);
 704                                 M_INTMOVE(s1, var->regoff);
 705                         }
 706                         break;
 707
 708                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 709                 case ICMD_DSTORE:     /* op1 = local variable                         */
 710
 711                         if ((src->varkind == LOCALVAR) &&
 712                             (src->varnum == iptr->op1)) {
 713                                 break;
 714                         }
 715                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 716                         if (var->flags & INMEMORY) {
 717                                 var_to_reg_flt(s1, src, REG_FTMP1);
 718                                 x86_64_movq_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 719
 720                         } else {
 721                                 var_to_reg_flt(s1, src, var->regoff);
 722                                 M_FLTMOVE(s1, var->regoff);
 723                         }
 724                         break;
 725
 726
 727                 /* pop/dup/swap operations ********************************************/
 728
 729                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 730
 731                 case ICMD_POP:        /* ..., value  ==> ...                          */
 732                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 733                         break;
 734
 735                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 736                         M_COPY(src, iptr->dst);
 737                         break;
 738
 739                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 740
 741                         M_COPY(src,       iptr->dst);
 742                         M_COPY(src->prev, iptr->dst->prev);
 743                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 744                         break;
 745
 746                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 747
 748                         M_COPY(src,             iptr->dst);
 749                         M_COPY(src->prev,       iptr->dst->prev);
 750                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 751                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 752                         break;
 753
 754                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 755
 756                         M_COPY(src,       iptr->dst);
 757                         M_COPY(src->prev, iptr->dst->prev);
 758                         break;
 759
 760                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 761
 762                         M_COPY(src,             iptr->dst);
 763                         M_COPY(src->prev,       iptr->dst->prev);
 764                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 765                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 766                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 767                         break;
 768
 769                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 770
 771                         M_COPY(src,                   iptr->dst);
 772                         M_COPY(src->prev,             iptr->dst->prev);
 773                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 774                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 775                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 776                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 777                         break;
 778
 779                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 780
 781                         M_COPY(src,       iptr->dst->prev);
 782                         M_COPY(src->prev, iptr->dst);
 783                         break;
 784
 785
 786                 /* integer operations *************************************************/
 787
 788                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 789
 790                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 791                         if (iptr->dst->flags & INMEMORY) {
 792                                 if (src->flags & INMEMORY) {
 793                                         if (src->regoff == iptr->dst->regoff) {
 794                                                 x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 795
 796                                         } else {
 797                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 798                                                 x86_64_negl_reg(cd, REG_ITMP1);
 799                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 800                                         }
 801
 802                                 } else {
 803                                         x86_64_movl_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 804                                         x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 805                                 }
 806
 807                         } else {
 808                                 if (src->flags & INMEMORY) {
 809                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 810                                         x86_64_negl_reg(cd, d);
 811
 812                                 } else {
 813                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 814                                         x86_64_negl_reg(cd, iptr->dst->regoff);
 815                                 }
 816                         }
 817                         break;
 818
 819                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 820
 821                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 822                         if (iptr->dst->flags & INMEMORY) {
 823                                 if (src->flags & INMEMORY) {
 824                                         if (src->regoff == iptr->dst->regoff) {
 825                                                 x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 826
 827                                         } else {
 828                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 829                                                 x86_64_neg_reg(cd, REG_ITMP1);
 830                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 831                                         }
 832
 833                                 } else {
 834                                         x86_64_mov_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 835                                         x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 836                                 }
 837
 838                         } else {
 839                                 if (src->flags & INMEMORY) {
 840                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 841                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 842
 843                                 } else {
 844                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 845                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 846                                 }
 847                         }
 848                         break;
 849
 850                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 851
 852                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 853                         if (src->flags & INMEMORY) {
 854                                 x86_64_movslq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 855
 856                         } else {
 857                                 x86_64_movslq_reg_reg(cd, src->regoff, d);
 858                         }
 859                         store_reg_to_var_int(iptr->dst, d);
 860                         break;
 861
 862                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 863
 864                         var_to_reg_int(s1, src, REG_ITMP1);
 865                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 866                         M_INTMOVE(s1, d);
 867                         store_reg_to_var_int(iptr->dst, d);
 868                         break;
 869
 870                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 871
 872                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 873                         if (src->flags & INMEMORY) {
 874                                 x86_64_movsbq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 875
 876                         } else {
 877                                 x86_64_movsbq_reg_reg(cd, src->regoff, d);
 878                         }
 879                         store_reg_to_var_int(iptr->dst, d);
 880                         break;
 881
 882                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 883
 884                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 885                         if (src->flags & INMEMORY) {
 886                                 x86_64_movzwq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 887
 888                         } else {
 889                                 x86_64_movzwq_reg_reg(cd, src->regoff, d);
 890                         }
 891                         store_reg_to_var_int(iptr->dst, d);
 892                         break;
 893
 894                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 895
 896                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 897                         if (src->flags & INMEMORY) {
 898                                 x86_64_movswq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 899
 900                         } else {
 901                                 x86_64_movswq_reg_reg(cd, src->regoff, d);
 902                         }
 903                         store_reg_to_var_int(iptr->dst, d);
 904                         break;
 905
 906
 907                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 908
 909                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 910                         x86_64_emit_ialu(cd, X86_64_ADD, src, iptr);
 911                         break;
 912
 913                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 914                                       /* val.i = constant                             */
 915
 916                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 917                         x86_64_emit_ialuconst(cd, X86_64_ADD, src, iptr);
 918                         break;
 919
 920                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 921
 922                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 923                         x86_64_emit_lalu(cd, X86_64_ADD, src, iptr);
 924                         break;
 925
 926                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 927                                       /* val.l = constant                             */
 928
 929                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 930                         x86_64_emit_laluconst(cd, X86_64_ADD, src, iptr);
 931                         break;
 932
 933                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 934
 935                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 936                         if (iptr->dst->flags & INMEMORY) {
 937                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 938                                         if (src->prev->regoff == iptr->dst->regoff) {
 939                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 940                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 941
 942                                         } else {
 943                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 944                                                 x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 945                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 946                                         }
 947
 948                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 949                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 950                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 951                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 952
 953                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 954                                         if (src->prev->regoff == iptr->dst->regoff) {
 955                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 956
 957                                         } else {
 958                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 959                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 960                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 961                                         }
 962
 963                                 } else {
 964                                         x86_64_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 965                                         x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 966                                 }
 967
 968                         } else {
 969                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 970                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 971                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 972
 973                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 974                                         M_INTMOVE(src->prev->regoff, d);
 975                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 976
 977                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 978                                         /* workaround for reg alloc */
 979                                         if (src->regoff == iptr->dst->regoff) {
 980                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 981                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 982                                                 M_INTMOVE(REG_ITMP1, d);
 983
 984                                         } else {
 985                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 986                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 987                                         }
 988
 989                                 } else {
 990                                         /* workaround for reg alloc */
 991                                         if (src->regoff == iptr->dst->regoff) {
 992                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
 993                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 994                                                 M_INTMOVE(REG_ITMP1, d);
 995
 996                                         } else {
 997                                                 M_INTMOVE(src->prev->regoff, d);
 998                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 999                                         }
1000                                 }
1001                         }
1002                         break;
1003
1004                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
1005                                       /* val.i = constant                             */
1006
1007                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1008                         x86_64_emit_ialuconst(cd, X86_64_SUB, src, iptr);
1009                         break;
1010
1011                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1012
1013                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1014                         if (iptr->dst->flags & INMEMORY) {
1015                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1016                                         if (src->prev->regoff == iptr->dst->regoff) {
1017                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1018                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1019
1020                                         } else {
1021                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1022                                                 x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1023                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1024                                         }
1025
1026                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1027                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1028                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1029                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1030
1031                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1032                                         if (src->prev->regoff == iptr->dst->regoff) {
1033                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1034
1035                                         } else {
1036                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1037                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1038                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1039                                         }
1040
1041                                 } else {
1042                                         x86_64_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
1043                                         x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1044                                 }
1045
1046                         } else {
1047                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1048                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1049                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1050
1051                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1052                                         M_INTMOVE(src->prev->regoff, d);
1053                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1054
1055                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1056                                         /* workaround for reg alloc */
1057                                         if (src->regoff == iptr->dst->regoff) {
1058                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1059                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1060                                                 M_INTMOVE(REG_ITMP1, d);
1061
1062                                         } else {
1063                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1064                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1065                                         }
1066
1067                                 } else {
1068                                         /* workaround for reg alloc */
1069                                         if (src->regoff == iptr->dst->regoff) {
1070                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1071                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1072                                                 M_INTMOVE(REG_ITMP1, d);
1073
1074                                         } else {
1075                                                 M_INTMOVE(src->prev->regoff, d);
1076                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1077                                         }
1078                                 }
1079                         }
1080                         break;
1081
1082                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1083                                       /* val.l = constant                             */
1084
1085                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1086                         x86_64_emit_laluconst(cd, X86_64_SUB, src, iptr);
1087                         break;
1088
1089                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1090
1091                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1092                         if (iptr->dst->flags & INMEMORY) {
1093                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1094                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1095                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1096                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1097
1098                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1099                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1100                                         x86_64_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1101                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1102
1103                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1104                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1105                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1106                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1107
1108                                 } else {
1109                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1110                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1111                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1112                                 }
1113
1114                         } else {
1115                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1116                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1117                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1118
1119                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1120                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1121                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1122
1123                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1124                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1125                                         x86_64_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1126
1127                                 } else {
1128                                         if (src->regoff == iptr->dst->regoff) {
1129                                                 x86_64_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1130
1131                                         } else {
1132                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1133                                                 x86_64_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1134                                         }
1135                                 }
1136                         }
1137                         break;
1138
1139                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1140                                       /* val.i = constant                             */
1141
1142                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1143                         if (iptr->dst->flags & INMEMORY) {
1144                                 if (src->flags & INMEMORY) {
1145                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1146                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1147
1148                                 } else {
1149                                         x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1150                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1151                                 }
1152
1153                         } else {
1154                                 if (src->flags & INMEMORY) {
1155                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1156
1157                                 } else {
1158                                         if (iptr->val.i == 2) {
1159                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1160                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1161
1162                                         } else {
1163                                                 x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1164                                         }
1165                                 }
1166                         }
1167                         break;
1168
1169                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1170
1171                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1172                         if (iptr->dst->flags & INMEMORY) {
1173                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1174                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1175                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1176                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1177
1178                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1179                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1180                                         x86_64_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1181                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1182
1183                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1184                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1185                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1186                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1187
1188                                 } else {
1189                                         x86_64_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1190                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1191                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1192                                 }
1193
1194                         } else {
1195                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1196                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1197                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1198
1199                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1200                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1201                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1202
1203                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1204                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1205                                         x86_64_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1206
1207                                 } else {
1208                                         if (src->regoff == iptr->dst->regoff) {
1209                                                 x86_64_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1210
1211                                         } else {
1212                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1213                                                 x86_64_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1214                                         }
1215                                 }
1216                         }
1217                         break;
1218
1219                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1220                                       /* val.l = constant                             */
1221
1222                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1223                         if (iptr->dst->flags & INMEMORY) {
1224                                 if (src->flags & INMEMORY) {
1225                                         if (IS_IMM32(iptr->val.l)) {
1226                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1227
1228                                         } else {
1229                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1230                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1231                                         }
1232                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1233
1234                                 } else {
1235                                         if (IS_IMM32(iptr->val.l)) {
1236                                                 x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1237
1238                                         } else {
1239                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1240                                                 x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1241                                         }
1242                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1243                                 }
1244
1245                         } else {
1246                                 if (src->flags & INMEMORY) {
1247                                         if (IS_IMM32(iptr->val.l)) {
1248                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1249
1250                                         } else {
1251                                                 x86_64_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1252                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1253                                         }
1254
1255                                 } else {
1256                                         /* should match in many cases */
1257                                         if (iptr->val.l == 2) {
1258                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1259                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1260
1261                                         } else {
1262                                                 if (IS_IMM32(iptr->val.l)) {
1263                                                         x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1264
1265                                                 } else {
1266                                                         x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1267                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1268                                                         x86_64_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1269                                                 }
1270                                         }
1271                                 }
1272                         }
1273                         break;
1274
1275                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1276
1277                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1278                 if (src->prev->flags & INMEMORY) {
1279                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1280
1281                         } else {
1282                                 M_INTMOVE(src->prev->regoff, RAX);
1283                         }
1284
1285                         if (src->flags & INMEMORY) {
1286                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1287
1288                         } else {
1289                                 M_INTMOVE(src->regoff, REG_ITMP3);
1290                         }
1291                         gen_div_check(src);
1292
1293                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1294                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1295                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1296                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                /* 6 bytes */
1297
1298                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1299                         x86_64_cltd(cd);
1300                         x86_64_idivl_reg(cd, REG_ITMP3);
1301
1302                         if (iptr->dst->flags & INMEMORY) {
1303                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1304                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1305
1306                         } else {
1307                                 M_INTMOVE(RAX, iptr->dst->regoff);
1308
1309                                 if (iptr->dst->regoff != RDX) {
1310                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1311                                 }
1312                         }
1313                         break;
1314
1315                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1316                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1317                         if (src->prev->flags & INMEMORY) {
1318                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1319
1320                         } else {
1321                                 M_INTMOVE(src->prev->regoff, RAX);
1322                         }
1323
1324                         if (src->flags & INMEMORY) {
1325                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1326
1327                         } else {
1328                                 M_INTMOVE(src->regoff, REG_ITMP3);
1329                         }
1330                         gen_div_check(src);
1331
1332                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1333
1334                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1335                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1336
1337
1338                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1339                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1340                         x86_64_jcc(cd, X86_64_CC_E, 1 + 3);                    /* 6 bytes */
1341
1342                         x86_64_cltd(cd);
1343                         x86_64_idivl_reg(cd, REG_ITMP3);
1344
1345                         if (iptr->dst->flags & INMEMORY) {
1346                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1347                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1348
1349                         } else {
1350                                 M_INTMOVE(RDX, iptr->dst->regoff);
1351
1352                                 if (iptr->dst->regoff != RDX) {
1353                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1354                                 }
1355                         }
1356                         break;
1357
1358                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1359                                       /* val.i = constant                             */
1360
1361                         var_to_reg_int(s1, src, REG_ITMP1);
1362                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1363                         M_INTMOVE(s1, REG_ITMP1);
1364                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1365                         x86_64_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1366                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1367                         x86_64_shiftl_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1368                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1369                         store_reg_to_var_int(iptr->dst, d);
1370                         break;
1371
1372                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1373                                       /* val.i = constant                             */
1374
1375                         var_to_reg_int(s1, src, REG_ITMP1);
1376                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1377                         M_INTMOVE(s1, REG_ITMP1);
1378                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1379                         x86_64_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1380                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1381                         x86_64_alul_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1382                         x86_64_alul_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1383                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1384                         store_reg_to_var_int(iptr->dst, d);
1385                         break;
1386
1387
1388                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1389
1390                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1391
1392                 if (src->prev->flags & INMEMORY) {
1393                                 M_LLD(RAX, REG_SP, src->prev->regoff * 8);
1394
1395                         } else {
1396                                 M_INTMOVE(src->prev->regoff, RAX);
1397                         }
1398
1399                         if (src->flags & INMEMORY) {
1400                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1401
1402                         } else {
1403                                 M_INTMOVE(src->regoff, REG_ITMP3);
1404                         }
1405                         gen_div_check(src);
1406
1407                         /* check as described in jvm spec */
1408                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1409                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, RAX);
1410                         M_BNE(4 + 6);
1411                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1412                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
1413
1414                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1415                         x86_64_cqto(cd);
1416                         x86_64_idiv_reg(cd, REG_ITMP3);
1417
1418                         if (iptr->dst->flags & INMEMORY) {
1419                                 M_LST(RAX, REG_SP, iptr->dst->regoff * 8);
1420                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1421
1422                         } else {
1423                                 M_INTMOVE(RAX, iptr->dst->regoff);
1424
1425                                 if (iptr->dst->regoff != RDX) {
1426                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1427                                 }
1428                         }
1429                         break;
1430
1431                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1432
1433                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1434                         if (src->prev->flags & INMEMORY) {
1435                                 M_LLD(REG_ITMP1, REG_SP, src->prev->regoff * 8);
1436
1437                         } else {
1438                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1439                         }
1440
1441                         if (src->flags & INMEMORY) {
1442                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1443
1444                         } else {
1445                                 M_INTMOVE(src->regoff, REG_ITMP3);
1446                         }
1447                         gen_div_check(src);
1448
1449                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1450
1451                         /* check as described in jvm spec */
1452                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1453                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP1);
1454                         M_BNE(3 + 4 + 6);
1455
1456 #if 0
1457                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1458 #endif
1459                         M_XOR(RDX, RDX);                                       /* 3 bytes */
1460                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1461                         M_BEQ(2 + 3);                                          /* 6 bytes */
1462
1463                         x86_64_cqto(cd);
1464                         x86_64_idiv_reg(cd, REG_ITMP3);
1465
1466                         if (iptr->dst->flags & INMEMORY) {
1467                                 M_LST(RDX, REG_SP, iptr->dst->regoff * 8);
1468                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1469
1470                         } else {
1471                                 M_INTMOVE(RDX, iptr->dst->regoff);
1472
1473                                 if (iptr->dst->regoff != RDX) {
1474                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1475                                 }
1476                         }
1477                         break;
1478
1479                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1480                                       /* val.i = constant                             */
1481
1482                         var_to_reg_int(s1, src, REG_ITMP1);
1483                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1484                         M_INTMOVE(s1, REG_ITMP1);
1485                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1486                         x86_64_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1487                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1488                         x86_64_shift_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1489                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1490                         store_reg_to_var_int(iptr->dst, d);
1491                         break;
1492
1493                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1494                                       /* val.l = constant                             */
1495
1496                         var_to_reg_int(s1, src, REG_ITMP1);
1497                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1498                         M_INTMOVE(s1, REG_ITMP1);
1499                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1500                         x86_64_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1501                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1502                         x86_64_alu_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1503                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1504                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1505                         store_reg_to_var_int(iptr->dst, d);
1506                         break;
1507
1508                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1509
1510                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1511                         x86_64_emit_ishift(cd, X86_64_SHL, src, iptr);
1512                         break;
1513
1514                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1515                                       /* val.i = constant                             */
1516
1517                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1518                         x86_64_emit_ishiftconst(cd, X86_64_SHL, src, iptr);
1519                         break;
1520
1521                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1522
1523                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1524                         x86_64_emit_ishift(cd, X86_64_SAR, src, iptr);
1525                         break;
1526
1527                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1528                                       /* val.i = constant                             */
1529
1530                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1531                         x86_64_emit_ishiftconst(cd, X86_64_SAR, src, iptr);
1532                         break;
1533
1534                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1535
1536                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1537                         x86_64_emit_ishift(cd, X86_64_SHR, src, iptr);
1538                         break;
1539
1540                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1541                                       /* val.i = constant                             */
1542
1543                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1544                         x86_64_emit_ishiftconst(cd, X86_64_SHR, src, iptr);
1545                         break;
1546
1547                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1548
1549                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1550                         x86_64_emit_lshift(cd, X86_64_SHL, src, iptr);
1551                         break;
1552
1553         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1554                                           /* val.i = constant                             */
1555
1556                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1557                         x86_64_emit_lshiftconst(cd, X86_64_SHL, src, iptr);
1558                         break;
1559
1560                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1561
1562                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1563                         x86_64_emit_lshift(cd, X86_64_SAR, src, iptr);
1564                         break;
1565
1566                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1567                                       /* val.i = constant                             */
1568
1569                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1570                         x86_64_emit_lshiftconst(cd, X86_64_SAR, src, iptr);
1571                         break;
1572
1573                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1574
1575                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1576                         x86_64_emit_lshift(cd, X86_64_SHR, src, iptr);
1577                         break;
1578
1579                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1580                                       /* val.l = constant                             */
1581
1582                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1583                         x86_64_emit_lshiftconst(cd, X86_64_SHR, src, iptr);
1584                         break;
1585
1586                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1587
1588                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1589                         x86_64_emit_ialu(cd, X86_64_AND, src, iptr);
1590                         break;
1591
1592                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1593                                       /* val.i = constant                             */
1594
1595                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1596                         x86_64_emit_ialuconst(cd, X86_64_AND, src, iptr);
1597                         break;
1598
1599                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1600
1601                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1602                         x86_64_emit_lalu(cd, X86_64_AND, src, iptr);
1603                         break;
1604
1605                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1606                                       /* val.l = constant                             */
1607
1608                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1609                         x86_64_emit_laluconst(cd, X86_64_AND, src, iptr);
1610                         break;
1611
1612                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1613
1614                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1615                         x86_64_emit_ialu(cd, X86_64_OR, src, iptr);
1616                         break;
1617
1618                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1619                                       /* val.i = constant                             */
1620
1621                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1622                         x86_64_emit_ialuconst(cd, X86_64_OR, src, iptr);
1623                         break;
1624
1625                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1626
1627                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1628                         x86_64_emit_lalu(cd, X86_64_OR, src, iptr);
1629                         break;
1630
1631                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1632                                       /* val.l = constant                             */
1633
1634                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1635                         x86_64_emit_laluconst(cd, X86_64_OR, src, iptr);
1636                         break;
1637
1638                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1639
1640                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1641                         x86_64_emit_ialu(cd, X86_64_XOR, src, iptr);
1642                         break;
1643
1644                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1645                                       /* val.i = constant                             */
1646
1647                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1648                         x86_64_emit_ialuconst(cd, X86_64_XOR, src, iptr);
1649                         break;
1650
1651                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1652
1653                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1654                         x86_64_emit_lalu(cd, X86_64_XOR, src, iptr);
1655                         break;
1656
1657                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1658                                       /* val.l = constant                             */
1659
1660                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1661                         x86_64_emit_laluconst(cd, X86_64_XOR, src, iptr);
1662                         break;
1663
1664
1665                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1666                                       /* op1 = variable, val.i = constant             */
1667
1668                         /* using inc and dec is definitely faster than add -- tested      */
1669                         /* with sieve                                                     */
1670
1671                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1672                         d = var->regoff;
1673                         if (var->flags & INMEMORY) {
1674                                 if (iptr->val.i == 1) {
1675                                         x86_64_incl_membase(cd, REG_SP, d * 8);
1676
1677                                 } else if (iptr->val.i == -1) {
1678                                         x86_64_decl_membase(cd, REG_SP, d * 8);
1679
1680                                 } else {
1681                                         x86_64_alul_imm_membase(cd, X86_64_ADD, iptr->val.i, REG_SP, d * 8);
1682                                 }
1683
1684                         } else {
1685                                 if (iptr->val.i == 1) {
1686                                         x86_64_incl_reg(cd, d);
1687
1688                                 } else if (iptr->val.i == -1) {
1689                                         x86_64_decl_reg(cd, d);
1690
1691                                 } else {
1692                                         x86_64_alul_imm_reg(cd, X86_64_ADD, iptr->val.i, d);
1693                                 }
1694                         }
1695                         break;
1696
1697
1698                 /* floating operations ************************************************/
1699
1700                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1701
1702                         var_to_reg_flt(s1, src, REG_FTMP1);
1703                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1704                         disp = dseg_adds4(cd, 0x80000000);
1705                         M_FLTMOVE(s1, d);
1706                         x86_64_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1707                         x86_64_xorps_reg_reg(cd, REG_FTMP2, d);
1708                         store_reg_to_var_flt(iptr->dst, d);
1709                         break;
1710
1711                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1712
1713                         var_to_reg_flt(s1, src, REG_FTMP1);
1714                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1715                         disp = dseg_adds8(cd, 0x8000000000000000);
1716                         M_FLTMOVE(s1, d);
1717                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1718                         x86_64_xorpd_reg_reg(cd, REG_FTMP2, d);
1719                         store_reg_to_var_flt(iptr->dst, d);
1720                         break;
1721
1722                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1723
1724                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1725                         var_to_reg_flt(s2, src, REG_FTMP2);
1726                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1727                         if (s1 == d) {
1728                                 x86_64_addss_reg_reg(cd, s2, d);
1729                         } else if (s2 == d) {
1730                                 x86_64_addss_reg_reg(cd, s1, d);
1731                         } else {
1732                                 M_FLTMOVE(s1, d);
1733                                 x86_64_addss_reg_reg(cd, s2, d);
1734                         }
1735                         store_reg_to_var_flt(iptr->dst, d);
1736                         break;
1737
1738                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1739
1740                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1741                         var_to_reg_flt(s2, src, REG_FTMP2);
1742                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1743                         if (s1 == d) {
1744                                 x86_64_addsd_reg_reg(cd, s2, d);
1745                         } else if (s2 == d) {
1746                                 x86_64_addsd_reg_reg(cd, s1, d);
1747                         } else {
1748                                 M_FLTMOVE(s1, d);
1749                                 x86_64_addsd_reg_reg(cd, s2, d);
1750                         }
1751                         store_reg_to_var_flt(iptr->dst, d);
1752                         break;
1753
1754                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1755
1756                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1757                         var_to_reg_flt(s2, src, REG_FTMP2);
1758                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1759                         if (s2 == d) {
1760                                 M_FLTMOVE(s2, REG_FTMP2);
1761                                 s2 = REG_FTMP2;
1762                         }
1763                         M_FLTMOVE(s1, d);
1764                         x86_64_subss_reg_reg(cd, s2, d);
1765                         store_reg_to_var_flt(iptr->dst, d);
1766                         break;
1767
1768                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1769
1770                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1771                         var_to_reg_flt(s2, src, REG_FTMP2);
1772                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1773                         if (s2 == d) {
1774                                 M_FLTMOVE(s2, REG_FTMP2);
1775                                 s2 = REG_FTMP2;
1776                         }
1777                         M_FLTMOVE(s1, d);
1778                         x86_64_subsd_reg_reg(cd, s2, d);
1779                         store_reg_to_var_flt(iptr->dst, d);
1780                         break;
1781
1782                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1783
1784                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1785                         var_to_reg_flt(s2, src, REG_FTMP2);
1786                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1787                         if (s1 == d) {
1788                                 x86_64_mulss_reg_reg(cd, s2, d);
1789                         } else if (s2 == d) {
1790                                 x86_64_mulss_reg_reg(cd, s1, d);
1791                         } else {
1792                                 M_FLTMOVE(s1, d);
1793                                 x86_64_mulss_reg_reg(cd, s2, d);
1794                         }
1795                         store_reg_to_var_flt(iptr->dst, d);
1796                         break;
1797
1798                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1799
1800                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1801                         var_to_reg_flt(s2, src, REG_FTMP2);
1802                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1803                         if (s1 == d) {
1804                                 x86_64_mulsd_reg_reg(cd, s2, d);
1805                         } else if (s2 == d) {
1806                                 x86_64_mulsd_reg_reg(cd, s1, d);
1807                         } else {
1808                                 M_FLTMOVE(s1, d);
1809                                 x86_64_mulsd_reg_reg(cd, s2, d);
1810                         }
1811                         store_reg_to_var_flt(iptr->dst, d);
1812                         break;
1813
1814                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1815
1816                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1817                         var_to_reg_flt(s2, src, REG_FTMP2);
1818                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1819                         if (s2 == d) {
1820                                 M_FLTMOVE(s2, REG_FTMP2);
1821                                 s2 = REG_FTMP2;
1822                         }
1823                         M_FLTMOVE(s1, d);
1824                         x86_64_divss_reg_reg(cd, s2, d);
1825                         store_reg_to_var_flt(iptr->dst, d);
1826                         break;
1827
1828                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1829
1830                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1831                         var_to_reg_flt(s2, src, REG_FTMP2);
1832                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1833                         if (s2 == d) {
1834                                 M_FLTMOVE(s2, REG_FTMP2);
1835                                 s2 = REG_FTMP2;
1836                         }
1837                         M_FLTMOVE(s1, d);
1838                         x86_64_divsd_reg_reg(cd, s2, d);
1839                         store_reg_to_var_flt(iptr->dst, d);
1840                         break;
1841
1842                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1843
1844                         var_to_reg_int(s1, src, REG_ITMP1);
1845                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1846                         x86_64_cvtsi2ss_reg_reg(cd, s1, d);
1847                         store_reg_to_var_flt(iptr->dst, d);
1848                         break;
1849
1850                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1851
1852                         var_to_reg_int(s1, src, REG_ITMP1);
1853                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1854                         x86_64_cvtsi2sd_reg_reg(cd, s1, d);
1855                         store_reg_to_var_flt(iptr->dst, d);
1856                         break;
1857
1858                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1859
1860                         var_to_reg_int(s1, src, REG_ITMP1);
1861                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1862                         x86_64_cvtsi2ssq_reg_reg(cd, s1, d);
1863                         store_reg_to_var_flt(iptr->dst, d);
1864                         break;
1865
1866                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1867
1868                         var_to_reg_int(s1, src, REG_ITMP1);
1869                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1870                         x86_64_cvtsi2sdq_reg_reg(cd, s1, d);
1871                         store_reg_to_var_flt(iptr->dst, d);
1872                         break;
1873
1874                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1875
1876                         var_to_reg_flt(s1, src, REG_FTMP1);
1877                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1878                         x86_64_cvttss2si_reg_reg(cd, s1, d);
1879                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1880                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1881                         x86_64_jcc(cd, X86_64_CC_NE, a);
1882                         M_FLTMOVE(s1, REG_FTMP1);
1883                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_f2i, REG_ITMP2);
1884                         x86_64_call_reg(cd, REG_ITMP2);
1885                         M_INTMOVE(REG_RESULT, d);
1886                         store_reg_to_var_int(iptr->dst, d);
1887                         break;
1888
1889                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1890
1891                         var_to_reg_flt(s1, src, REG_FTMP1);
1892                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1893                         x86_64_cvttsd2si_reg_reg(cd, s1, d);
1894                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1895                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1896                         x86_64_jcc(cd, X86_64_CC_NE, a);
1897                         M_FLTMOVE(s1, REG_FTMP1);
1898                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_d2i, REG_ITMP2);
1899                         x86_64_call_reg(cd, REG_ITMP2);
1900                         M_INTMOVE(REG_RESULT, d);
1901                         store_reg_to_var_int(iptr->dst, d);
1902                         break;
1903
1904                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1905
1906                         var_to_reg_flt(s1, src, REG_FTMP1);
1907                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1908                         x86_64_cvttss2siq_reg_reg(cd, s1, d);
1909                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1910                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1911                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1912                         x86_64_jcc(cd, X86_64_CC_NE, a);
1913                         M_FLTMOVE(s1, REG_FTMP1);
1914                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_f2l, REG_ITMP2);
1915                         x86_64_call_reg(cd, REG_ITMP2);
1916                         M_INTMOVE(REG_RESULT, d);
1917                         store_reg_to_var_int(iptr->dst, d);
1918                         break;
1919
1920                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1921
1922                         var_to_reg_flt(s1, src, REG_FTMP1);
1923                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1924                         x86_64_cvttsd2siq_reg_reg(cd, s1, d);
1925                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1926                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1927                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1928                         x86_64_jcc(cd, X86_64_CC_NE, a);
1929                         M_FLTMOVE(s1, REG_FTMP1);
1930                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_d2l, REG_ITMP2);
1931                         x86_64_call_reg(cd, REG_ITMP2);
1932                         M_INTMOVE(REG_RESULT, d);
1933                         store_reg_to_var_int(iptr->dst, d);
1934                         break;
1935
1936                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1937
1938                         var_to_reg_flt(s1, src, REG_FTMP1);
1939                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1940                         x86_64_cvtss2sd_reg_reg(cd, s1, d);
1941                         store_reg_to_var_flt(iptr->dst, d);
1942                         break;
1943
1944                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1945
1946                         var_to_reg_flt(s1, src, REG_FTMP1);
1947                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1948                         x86_64_cvtsd2ss_reg_reg(cd, s1, d);
1949                         store_reg_to_var_flt(iptr->dst, d);
1950                         break;
1951
1952                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1953                                           /* == => 0, < => 1, > => -1 */
1954
1955                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1956                         var_to_reg_flt(s2, src, REG_FTMP2);
1957                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1958                         M_CLR(d);
1959                         M_MOV_IMM(1, REG_ITMP1);
1960                         M_MOV_IMM(-1, REG_ITMP2);
1961                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1962                         M_CMOVB(REG_ITMP1, d);
1963                         M_CMOVA(REG_ITMP2, d);
1964                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1965                         store_reg_to_var_int(iptr->dst, d);
1966                         break;
1967
1968                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1969                                           /* == => 0, < => 1, > => -1 */
1970
1971                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1972                         var_to_reg_flt(s2, src, REG_FTMP2);
1973                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1974                         M_CLR(d);
1975                         M_MOV_IMM(1, REG_ITMP1);
1976                         M_MOV_IMM(-1, REG_ITMP2);
1977                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1978                         M_CMOVB(REG_ITMP1, d);
1979                         M_CMOVA(REG_ITMP2, d);
1980                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1981                         store_reg_to_var_int(iptr->dst, d);
1982                         break;
1983
1984                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1985                                           /* == => 0, < => 1, > => -1 */
1986
1987                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1988                         var_to_reg_flt(s2, src, REG_FTMP2);
1989                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1990                         M_CLR(d);
1991                         M_MOV_IMM(1, REG_ITMP1);
1992                         M_MOV_IMM(-1, REG_ITMP2);
1993                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1994                         M_CMOVB(REG_ITMP1, d);
1995                         M_CMOVA(REG_ITMP2, d);
1996                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1997                         store_reg_to_var_int(iptr->dst, d);
1998                         break;
1999
2000                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
2001                                           /* == => 0, < => 1, > => -1 */
2002
2003                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
2004                         var_to_reg_flt(s2, src, REG_FTMP2);
2005                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2006                         M_CLR(d);
2007                         M_MOV_IMM(1, REG_ITMP1);
2008                         M_MOV_IMM(-1, REG_ITMP2);
2009                         x86_64_ucomisd_reg_reg(cd, s1, s2);
2010                         M_CMOVB(REG_ITMP1, d);
2011                         M_CMOVA(REG_ITMP2, d);
2012                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
2013                         store_reg_to_var_int(iptr->dst, d);
2014                         break;
2015
2016
2017                 /* memory operations **************************************************/
2018
2019                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
2020
2021                         var_to_reg_int(s1, src, REG_ITMP1);
2022                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2023                         gen_nullptr_check(s1);
2024                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
2025                         store_reg_to_var_int(iptr->dst, d);
2026                         break;
2027
2028                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2029
2030                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2031                         var_to_reg_int(s2, src, REG_ITMP2);
2032                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2033                         if (iptr->op1 == 0) {
2034                                 gen_nullptr_check(s1);
2035                                 gen_bound_check;
2036                         }
2037                         x86_64_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2038                         store_reg_to_var_int(iptr->dst, d);
2039                         break;
2040
2041                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2042
2043                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2044                         var_to_reg_int(s2, src, REG_ITMP2);
2045                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2046                         if (iptr->op1 == 0) {
2047                                 gen_nullptr_check(s1);
2048                                 gen_bound_check;
2049                         }
2050                         x86_64_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2051                         store_reg_to_var_int(iptr->dst, d);
2052                         break;
2053
2054                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2055
2056                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2057                         var_to_reg_int(s2, src, REG_ITMP2);
2058                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2059                         if (iptr->op1 == 0) {
2060                                 gen_nullptr_check(s1);
2061                                 gen_bound_check;
2062                         }
2063                         x86_64_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2064                         store_reg_to_var_int(iptr->dst, d);
2065                         break;
2066
2067                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
2068
2069                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2070                         var_to_reg_int(s2, src, REG_ITMP2);
2071                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2072                         if (iptr->op1 == 0) {
2073                                 gen_nullptr_check(s1);
2074                                 gen_bound_check;
2075                         }
2076                         x86_64_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
2077                         store_reg_to_var_int(iptr->dst, d);
2078                         break;
2079
2080                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
2081
2082                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2083                         var_to_reg_int(s2, src, REG_ITMP2);
2084                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2085                         if (iptr->op1 == 0) {
2086                                 gen_nullptr_check(s1);
2087                                 gen_bound_check;
2088                         }
2089                         x86_64_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
2090                         store_reg_to_var_int(iptr->dst, d);
2091                         break;
2092
2093                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
2094
2095                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2096                         var_to_reg_int(s2, src, REG_ITMP2);
2097                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
2098                         if (iptr->op1 == 0) {
2099                                 gen_nullptr_check(s1);
2100                                 gen_bound_check;
2101                         }
2102                         x86_64_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2103                         store_reg_to_var_flt(iptr->dst, d);
2104                         break;
2105
2106                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2107
2108                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2109                         var_to_reg_int(s2, src, REG_ITMP2);
2110                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
2111                         if (iptr->op1 == 0) {
2112                                 gen_nullptr_check(s1);
2113                                 gen_bound_check;
2114                         }
2115                         x86_64_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2116                         store_reg_to_var_flt(iptr->dst, d);
2117                         break;
2118
2119                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
2120
2121                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2122                         var_to_reg_int(s2, src, REG_ITMP2);
2123                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2124                         if (iptr->op1 == 0) {
2125                                 gen_nullptr_check(s1);
2126                                 gen_bound_check;
2127                         }
2128                         x86_64_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
2129                         store_reg_to_var_int(iptr->dst, d);
2130                         break;
2131
2132
2133                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2134
2135                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2136                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2137                         if (iptr->op1 == 0) {
2138                                 gen_nullptr_check(s1);
2139                                 gen_bound_check;
2140                         }
2141                         var_to_reg_int(s3, src, REG_ITMP3);
2142                         x86_64_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2143                         break;
2144
2145                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2146
2147                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2148                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2149                         if (iptr->op1 == 0) {
2150                                 gen_nullptr_check(s1);
2151                                 gen_bound_check;
2152                         }
2153                         var_to_reg_int(s3, src, REG_ITMP3);
2154                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2155                         break;
2156
2157                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2158
2159                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2160                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2161                         if (iptr->op1 == 0) {
2162                                 gen_nullptr_check(s1);
2163                                 gen_bound_check;
2164                         }
2165                         var_to_reg_int(s3, src, REG_ITMP3);
2166                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2167                         break;
2168
2169                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2170
2171                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2172                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2173                         if (iptr->op1 == 0) {
2174                                 gen_nullptr_check(s1);
2175                                 gen_bound_check;
2176                         }
2177                         var_to_reg_int(s3, src, REG_ITMP3);
2178                         x86_64_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2179                         break;
2180
2181                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2182
2183                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2184                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2185                         if (iptr->op1 == 0) {
2186                                 gen_nullptr_check(s1);
2187                                 gen_bound_check;
2188                         }
2189                         var_to_reg_int(s3, src, REG_ITMP3);
2190                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2191                         break;
2192
2193                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2194
2195                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2196                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2197                         if (iptr->op1 == 0) {
2198                                 gen_nullptr_check(s1);
2199                                 gen_bound_check;
2200                         }
2201                         var_to_reg_flt(s3, src, REG_FTMP3);
2202                         x86_64_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2203                         break;
2204
2205                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2206
2207                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2208                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2209                         if (iptr->op1 == 0) {
2210                                 gen_nullptr_check(s1);
2211                                 gen_bound_check;
2212                         }
2213                         var_to_reg_flt(s3, src, REG_FTMP3);
2214                         x86_64_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2215                         break;
2216
2217                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2218
2219                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2220                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2221                         if (iptr->op1 == 0) {
2222                                 gen_nullptr_check(s1);
2223                                 gen_bound_check;
2224                         }
2225                         var_to_reg_int(s3, src, REG_ITMP3);
2226
2227                         M_MOV(s1, rd->argintregs[0]);
2228                         M_MOV(s3, rd->argintregs[1]);
2229                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
2230                         M_CALL(REG_ITMP1);
2231                         M_TEST(REG_RESULT);
2232                         M_BEQ(0);
2233                         codegen_add_arraystoreexception_ref(cd, cd->mcodeptr);
2234
2235                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2236                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2237                         var_to_reg_int(s3, src, REG_ITMP3);
2238                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2239                         break;
2240
2241
2242                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2243
2244                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2245                         var_to_reg_int(s2, src, REG_ITMP2);
2246                         if (iptr->op1 == 0) {
2247                                 gen_nullptr_check(s1);
2248                                 gen_bound_check;
2249                         }
2250                         x86_64_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2251                         break;
2252
2253                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2254
2255                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2256                         var_to_reg_int(s2, src, REG_ITMP2);
2257                         if (iptr->op1 == 0) {
2258                                 gen_nullptr_check(s1);
2259                                 gen_bound_check;
2260                         }
2261                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2262                         break;
2263
2264                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2265
2266                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2267                         var_to_reg_int(s2, src, REG_ITMP2);
2268                         if (iptr->op1 == 0) {
2269                                 gen_nullptr_check(s1);
2270                                 gen_bound_check;
2271                         }
2272                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2273                         break;
2274
2275                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2276
2277                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2278                         var_to_reg_int(s2, src, REG_ITMP2);
2279                         if (iptr->op1 == 0) {
2280                                 gen_nullptr_check(s1);
2281                                 gen_bound_check;
2282                         }
2283                         x86_64_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2284                         break;
2285
2286                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2287
2288                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2289                         var_to_reg_int(s2, src, REG_ITMP2);
2290                         if (iptr->op1 == 0) {
2291                                 gen_nullptr_check(s1);
2292                                 gen_bound_check;
2293                         }
2294
2295                         if (IS_IMM32(iptr->val.l)) {
2296                                 x86_64_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2297                         } else {
2298                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2299                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2300                         }
2301                         break;
2302
2303                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2304
2305                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2306                         var_to_reg_int(s2, src, REG_ITMP2);
2307                         if (iptr->op1 == 0) {
2308                                 gen_nullptr_check(s1);
2309                                 gen_bound_check;
2310                         }
2311                         x86_64_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2312                         break;
2313
2314
2315                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2316                                       /* op1 = type, val.a = field address            */
2317
2318                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2319                                 disp = dseg_addaddress(cd, NULL);
2320
2321 /*                              PROFILE_CYCLE_STOP; */
2322
2323                                 codegen_addpatchref(cd, cd->mcodeptr,
2324                                                                         PATCHER_get_putstatic,
2325                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), disp);
2326
2327                                 if (opt_showdisassemble) {
2328                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2329                                 }
2330
2331 /*                              PROFILE_CYCLE_START; */
2332
2333                         } else {
2334                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr);
2335
2336                                 disp = dseg_addaddress(cd, &(fi->value));
2337
2338                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2339                                         PROFILE_CYCLE_STOP;
2340
2341                                         codegen_addpatchref(cd, cd->mcodeptr,
2342                                                                                 PATCHER_clinit, fi->class, 0);
2343
2344                                         if (opt_showdisassemble) {
2345                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2346                                         }
2347
2348                                         PROFILE_CYCLE_START;
2349                                 }
2350                         }
2351
2352                         /* This approach is much faster than moving the field
2353                            address inline into a register. */
2354
2355                         M_ALD(REG_ITMP2, RIP, -(((ptrint) cd->mcodeptr + 7) -
2356                                                                         (ptrint) cd->mcodebase) + disp);
2357
2358                         switch (iptr->op1) {
2359                         case TYPE_INT:
2360                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2361                                 M_ILD(d, REG_ITMP2, 0);
2362                                 store_reg_to_var_int(iptr->dst, d);
2363                                 break;
2364                         case TYPE_LNG:
2365                         case TYPE_ADR:
2366                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2367                                 M_LLD(d, REG_ITMP2, 0);
2368                                 store_reg_to_var_int(iptr->dst, d);
2369                                 break;
2370                         case TYPE_FLT:
2371                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2372                                 x86_64_movss_membase_reg(cd, REG_ITMP2, 0, d);
2373                                 store_reg_to_var_flt(iptr->dst, d);
2374                                 break;
2375                         case TYPE_DBL:
2376                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2377                                 x86_64_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2378                                 store_reg_to_var_flt(iptr->dst, d);
2379                                 break;
2380                         }
2381                         break;
2382
2383                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2384                                       /* op1 = type, val.a = field address            */
2385
2386                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2387                                 disp = dseg_addaddress(cd, NULL);
2388
2389 /*                              PROFILE_CYCLE_STOP; */
2390
2391                                 codegen_addpatchref(cd, cd->mcodeptr,
2392                                                                         PATCHER_get_putstatic,
2393                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), disp);
2394
2395                                 if (opt_showdisassemble) {
2396                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2397                                 }
2398
2399 /*                              PROFILE_CYCLE_START; */
2400
2401                         } else {
2402                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr);
2403
2404                                 disp = dseg_addaddress(cd, &(fi->value));
2405
2406                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2407                                         PROFILE_CYCLE_STOP;
2408
2409                                         codegen_addpatchref(cd, cd->mcodeptr,
2410                                                                                 PATCHER_clinit, fi->class, 0);
2411
2412                                         if (opt_showdisassemble) {
2413                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2414                                         }
2415
2416                                         PROFILE_CYCLE_START;
2417                                 }
2418                         }
2419
2420                         /* This approach is much faster than moving the field
2421                            address inline into a register. */
2422
2423                         M_ALD(REG_ITMP2, RIP, -(((ptrint) cd->mcodeptr + 7) -
2424                                                                         (ptrint) cd->mcodebase) + disp);
2425
2426                         switch (iptr->op1) {
2427                         case TYPE_INT:
2428                                 var_to_reg_int(s2, src, REG_ITMP1);
2429                                 M_IST(s2, REG_ITMP2, 0);
2430                                 break;
2431                         case TYPE_LNG:
2432                         case TYPE_ADR:
2433                                 var_to_reg_int(s2, src, REG_ITMP1);
2434                                 M_LST(s2, REG_ITMP2, 0);
2435                                 break;
2436                         case TYPE_FLT:
2437                                 var_to_reg_flt(s2, src, REG_FTMP1);
2438                                 x86_64_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2439                                 break;
2440                         case TYPE_DBL:
2441                                 var_to_reg_flt(s2, src, REG_FTMP1);
2442                                 x86_64_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2443                                 break;
2444                         }
2445                         break;
2446
2447                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2448                                           /* val = value (in current instruction)     */
2449                                           /* op1 = type, val.a = field address (in    */
2450                                           /* following NOP)                           */
2451
2452                         if (INSTRUCTION_IS_UNRESOLVED(iptr + 1)) {
2453                                 disp = dseg_addaddress(cd, NULL);
2454
2455 /*                              PROFILE_CYCLE_STOP; */
2456
2457                                 codegen_addpatchref(cd, cd->mcodeptr,
2458                                                                         PATCHER_get_putstatic,
2459                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr + 1), disp);
2460
2461                                 if (opt_showdisassemble) {
2462                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2463                                 }
2464
2465 /*                              PROFILE_CYCLE_START; */
2466
2467                         } else {
2468                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr + 1);
2469
2470                                 disp = dseg_addaddress(cd, &(fi->value));
2471
2472                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2473                                         PROFILE_CYCLE_STOP;
2474
2475                                         codegen_addpatchref(cd, cd->mcodeptr,
2476                                                                                 PATCHER_clinit, fi->class, 0);
2477
2478                                         if (opt_showdisassemble) {
2479                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2480                                         }
2481
2482                                         PROFILE_CYCLE_START;
2483                                 }
2484                         }
2485
2486                         /* This approach is much faster than moving the field
2487                            address inline into a register. */
2488
2489                         M_ALD(REG_ITMP1, RIP, -(((ptrint) cd->mcodeptr + 7) -
2490                                                                         (ptrint) cd->mcodebase) + disp);
2491
2492                         switch (iptr->op1) {
2493                         case TYPE_INT:
2494                         case TYPE_FLT:
2495                                 M_IST_IMM(iptr->val.i, REG_ITMP1, 0);
2496                                 break;
2497                         case TYPE_LNG:
2498                         case TYPE_ADR:
2499                         case TYPE_DBL:
2500                                 if (IS_IMM32(iptr->val.l)) {
2501                                         M_LST_IMM32(iptr->val.l, REG_ITMP1, 0);
2502                                 } else {
2503                                         M_IST_IMM(iptr->val.l, REG_ITMP1, 0);
2504                                         M_IST_IMM(iptr->val.l >> 32, REG_ITMP1, 4);
2505                                 }
2506                                 break;
2507                         }
2508                         break;
2509
2510                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2511                                       /* op1 = type, val.i = field offset             */
2512
2513                         var_to_reg_int(s1, src, REG_ITMP1);
2514                         gen_nullptr_check(s1);
2515
2516                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2517 /*                              PROFILE_CYCLE_STOP; */
2518
2519                                 codegen_addpatchref(cd, cd->mcodeptr,
2520                                                                         PATCHER_get_putfield,
2521                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), 0);
2522
2523                                 if (opt_showdisassemble) {
2524                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2525                                 }
2526
2527 /*                              PROFILE_CYCLE_START; */
2528
2529                                 disp = 0;
2530
2531                         } else {
2532                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr)->offset;
2533                         }
2534
2535                         switch (iptr->op1) {
2536                         case TYPE_INT:
2537                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2538                                 if (INSTRUCTION_IS_UNRESOLVED(iptr))
2539                                         M_ILD32(d, s1, disp);
2540                                 else
2541                                         M_ILD(d, s1, disp);
2542                                 store_reg_to_var_int(iptr->dst, d);
2543                                 break;
2544                         case TYPE_LNG:
2545                         case TYPE_ADR:
2546                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2547                                 if (INSTRUCTION_IS_UNRESOLVED(iptr))
2548                                         M_LLD32(d, s1, disp);
2549                                 else
2550                                         M_LLD(d, s1, disp);
2551                                 store_reg_to_var_int(iptr->dst, d);
2552                                 break;
2553                         case TYPE_FLT:
2554                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2555                                 x86_64_movss_membase32_reg(cd, s1, disp, d);
2556                                 store_reg_to_var_flt(iptr->dst, d);
2557                                 break;
2558                         case TYPE_DBL:
2559                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2560                                 x86_64_movsd_membase32_reg(cd, s1, disp, d);
2561                                 store_reg_to_var_flt(iptr->dst, d);
2562                                 break;
2563                         }
2564                         break;
2565
2566                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2567                                       /* op1 = type, val.i = field offset             */
2568
2569                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2570                         gen_nullptr_check(s1);
2571
2572                         if (IS_INT_LNG_TYPE(iptr->op1)) {
2573                                 var_to_reg_int(s2, src, REG_ITMP2);
2574                         } else {
2575                                 var_to_reg_flt(s2, src, REG_FTMP2);
2576                         }
2577
2578                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2579 /*                              PROFILE_CYCLE_STOP; */
2580
2581                                 codegen_addpatchref(cd, cd->mcodeptr,
2582                                                                         PATCHER_get_putfield,
2583                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), 0);
2584
2585                                 if (opt_showdisassemble) {
2586                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2587                                 }
2588
2589 /*                              PROFILE_CYCLE_START; */
2590
2591                                 disp = 0;
2592
2593                         } else {
2594                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr)->offset;
2595                         }
2596
2597                         switch (iptr->op1) {
2598                         case TYPE_INT:
2599                                 if (INSTRUCTION_IS_UNRESOLVED(iptr))
2600                                         M_IST32(s2, s1, disp);
2601                                 else
2602                                         M_IST(s2, s1, disp);
2603                                 break;
2604                         case TYPE_LNG:
2605                         case TYPE_ADR:
2606                                 if (INSTRUCTION_IS_UNRESOLVED(iptr))
2607                                         M_LST32(s2, s1, disp);
2608                                 else
2609                                         M_LST(s2, s1, disp);
2610                                 break;
2611                         case TYPE_FLT:
2612                                 x86_64_movss_reg_membase32(cd, s2, s1, disp);
2613                                 break;
2614                         case TYPE_DBL:
2615                                 x86_64_movsd_reg_membase32(cd, s2, s1, disp);
2616                                 break;
2617                         }
2618                         break;
2619
2620                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2621                                           /* val = value (in current instruction)     */
2622                                           /* op1 = type, val.a = field address (in    */
2623                                           /* following NOP)                           */
2624
2625                         var_to_reg_int(s1, src, REG_ITMP1);
2626                         gen_nullptr_check(s1);
2627
2628                         if (INSTRUCTION_IS_UNRESOLVED(iptr + 1)) {
2629 /*                              PROFILE_CYCLE_STOP; */
2630
2631                                 codegen_addpatchref(cd, cd->mcodeptr,
2632                                                                         PATCHER_putfieldconst,
2633                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr + 1), 0);
2634
2635                                 if (opt_showdisassemble) {
2636                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2637                                 }
2638
2639 /*                              PROFILE_CYCLE_START; */
2640
2641                                 disp = 0;
2642
2643                         } else {
2644                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr + 1)->offset;
2645                         }
2646
2647                         switch (iptr->op1) {
2648                         case TYPE_INT:
2649                         case TYPE_FLT:
2650                                 if (INSTRUCTION_IS_UNRESOLVED(iptr + 1))
2651                                         M_IST32_IMM(iptr->val.i, s1, disp);
2652                                 else
2653                                         M_IST_IMM(iptr->val.i, s1, disp);
2654                                 break;
2655                         case TYPE_LNG:
2656                         case TYPE_ADR:
2657                         case TYPE_DBL:
2658                                 /* We can only optimize the move, if the class is
2659                                    resolved. Otherwise we don't know what to patch. */
2660                                 if (INSTRUCTION_IS_UNRESOLVED(iptr + 1)) {
2661                                         M_IST32_IMM(iptr->val.l, s1, disp);
2662                                         M_IST32_IMM(iptr->val.l >> 32, s1, disp + 4);
2663                                 } else {
2664                                         if (IS_IMM32(iptr->val.l)) {
2665                                                 M_LST_IMM32(iptr->val.l, s1, disp);
2666                                         } else {
2667                                                 M_IST_IMM(iptr->val.l, s1, disp);
2668                                                 M_IST_IMM(iptr->val.l >> 32, s1, disp + 4);
2669                                         }
2670                                 }
2671                                 break;
2672                         }
2673                         break;
2674
2675
2676                 /* branch operations **************************************************/
2677
2678                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2679
2680                         var_to_reg_int(s1, src, REG_ITMP1);
2681                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2682
2683                         PROFILE_CYCLE_STOP;
2684
2685 #ifdef ENABLE_VERIFIER
2686                         if (iptr->val.a) {
2687                                 codegen_addpatchref(cd, cd->mcodeptr,
2688                                                                         PATCHER_athrow_areturn,
2689                                                                         (unresolved_class *) iptr->val.a, 0);
2690
2691                                 if (opt_showdisassemble) {
2692                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2693                                 }
2694                         }
2695 #endif /* ENABLE_VERIFIER */
2696
2697                         M_CALL_IMM(0);                            /* passing exception pc */
2698                         M_POP(REG_ITMP2_XPC);
2699
2700                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2701                         M_JMP(REG_ITMP3);
2702                         break;
2703
2704                 case ICMD_GOTO:         /* ... ==> ...                                */
2705                                         /* op1 = target JavaVM pc                     */
2706
2707                         M_JMP_IMM(0);
2708                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2709                         break;
2710
2711                 case ICMD_JSR:          /* ... ==> ...                                */
2712                                         /* op1 = target JavaVM pc                     */
2713
2714                         M_CALL_IMM(0);
2715                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2716                         break;
2717
2718                 case ICMD_RET:          /* ... ==> ...                                */
2719                                         /* op1 = local variable                       */
2720
2721                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2722                         var_to_reg_int(s1, var, REG_ITMP1);
2723                         M_JMP(s1);
2724                         break;
2725
2726                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2727                                         /* op1 = target JavaVM pc                     */
2728
2729                         if (src->flags & INMEMORY)
2730                                 M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
2731                         else
2732                                 M_TEST(src->regoff);
2733                         M_BEQ(0);
2734                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2735                         break;
2736
2737                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2738                                         /* op1 = target JavaVM pc                     */
2739
2740                         if (src->flags & INMEMORY)
2741                                 M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
2742                         else
2743                                 M_TEST(src->regoff);
2744                         M_BNE(0);
2745                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2746                         break;
2747
2748                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2749                                         /* op1 = target JavaVM pc, val.i = constant   */
2750
2751                         x86_64_emit_ifcc(cd, X86_64_CC_E, src, iptr);
2752                         break;
2753
2754                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2755                                         /* op1 = target JavaVM pc, val.i = constant   */
2756
2757                         x86_64_emit_ifcc(cd, X86_64_CC_L, src, iptr);
2758                         break;
2759
2760                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2761                                         /* op1 = target JavaVM pc, val.i = constant   */
2762
2763                         x86_64_emit_ifcc(cd, X86_64_CC_LE, src, iptr);
2764                         break;
2765
2766                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2767                                         /* op1 = target JavaVM pc, val.i = constant   */
2768
2769                         x86_64_emit_ifcc(cd, X86_64_CC_NE, src, iptr);
2770                         break;
2771
2772                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2773                                         /* op1 = target JavaVM pc, val.i = constant   */
2774
2775                         x86_64_emit_ifcc(cd, X86_64_CC_G, src, iptr);
2776                         break;
2777
2778                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2779                                         /* op1 = target JavaVM pc, val.i = constant   */
2780
2781                         x86_64_emit_ifcc(cd, X86_64_CC_GE, src, iptr);
2782                         break;
2783
2784                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2785                                         /* op1 = target JavaVM pc, val.l = constant   */
2786
2787                         x86_64_emit_if_lcc(cd, X86_64_CC_E, src, iptr);
2788                         break;
2789
2790                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2791                                         /* op1 = target JavaVM pc, val.l = constant   */
2792
2793                         x86_64_emit_if_lcc(cd, X86_64_CC_L, src, iptr);
2794                         break;
2795
2796                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2797                                         /* op1 = target JavaVM pc, val.l = constant   */
2798
2799                         x86_64_emit_if_lcc(cd, X86_64_CC_LE, src, iptr);
2800                         break;
2801
2802                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2803                                         /* op1 = target JavaVM pc, val.l = constant   */
2804
2805                         x86_64_emit_if_lcc(cd, X86_64_CC_NE, src, iptr);
2806                         break;
2807
2808                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2809                                         /* op1 = target JavaVM pc, val.l = constant   */
2810
2811                         x86_64_emit_if_lcc(cd, X86_64_CC_G, src, iptr);
2812                         break;
2813
2814                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2815                                         /* op1 = target JavaVM pc, val.l = constant   */
2816
2817                         x86_64_emit_if_lcc(cd, X86_64_CC_GE, src, iptr);
2818                         break;
2819
2820                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2821                                         /* op1 = target JavaVM pc                     */
2822
2823                         x86_64_emit_if_icmpcc(cd, X86_64_CC_E, src, iptr);
2824                         break;
2825
2826                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2827                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2828
2829                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_E, src, iptr);
2830                         break;
2831
2832                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2833                                         /* op1 = target JavaVM pc                     */
2834
2835                         x86_64_emit_if_icmpcc(cd, X86_64_CC_NE, src, iptr);
2836                         break;
2837
2838                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2839                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2840
2841                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_NE, src, iptr);
2842                         break;
2843
2844                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2845                                         /* op1 = target JavaVM pc                     */
2846
2847                         x86_64_emit_if_icmpcc(cd, X86_64_CC_L, src, iptr);
2848                         break;
2849
2850                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2851                                     /* op1 = target JavaVM pc                     */
2852
2853                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_L, src, iptr);
2854                         break;
2855
2856                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2857                                         /* op1 = target JavaVM pc                     */
2858
2859                         x86_64_emit_if_icmpcc(cd, X86_64_CC_G, src, iptr);
2860                         break;
2861
2862                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2863                                 /* op1 = target JavaVM pc                     */
2864
2865                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_G, src, iptr);
2866                         break;
2867
2868                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2869                                         /* op1 = target JavaVM pc                     */
2870
2871                         x86_64_emit_if_icmpcc(cd, X86_64_CC_LE, src, iptr);
2872                         break;
2873
2874                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2875                                         /* op1 = target JavaVM pc                     */
2876
2877                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_LE, src, iptr);
2878                         break;
2879
2880                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2881                                         /* op1 = target JavaVM pc                     */
2882
2883                         x86_64_emit_if_icmpcc(cd, X86_64_CC_GE, src, iptr);
2884                         break;
2885
2886                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2887                                     /* op1 = target JavaVM pc                     */
2888
2889                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_GE, src, iptr);
2890                         break;
2891
2892                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2893
2894                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2895                         break;
2896
2897                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2898                 case ICMD_IFNE_ICONST:  /* val.i = constant                           */
2899                 case ICMD_IFLT_ICONST:
2900                 case ICMD_IFGE_ICONST:
2901                 case ICMD_IFGT_ICONST:
2902                 case ICMD_IFLE_ICONST:
2903
2904                         var_to_reg_int(s1, src, REG_ITMP1);
2905                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2906                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2907                                 if (s1 == d) {
2908                                         M_INTMOVE(s1, REG_ITMP1);
2909                                         s1 = REG_ITMP1;
2910                                 }
2911                                 if (iptr[1].val.i == 0)
2912                                         M_CLR(d);
2913                                 else
2914                                         M_IMOV_IMM(iptr[1].val.i, d);
2915                         }
2916                         if (iptr->val.i == 0)
2917                                 M_CLR(REG_ITMP2);
2918                         else
2919                                 M_IMOV_IMM(iptr->val.i, REG_ITMP2);
2920                         M_ITEST(s1);
2921
2922                         switch (iptr->opc) {
2923                         case ICMD_IFEQ_ICONST:
2924                                 M_CMOVEQ(REG_ITMP2, d);
2925                                 break;
2926                         case ICMD_IFNE_ICONST:
2927                                 M_CMOVNE(REG_ITMP2, d);
2928                                 break;
2929                         case ICMD_IFLT_ICONST:
2930                                 M_CMOVLT(REG_ITMP2, d);
2931                                 break;
2932                         case ICMD_IFGE_ICONST:
2933                                 M_CMOVGE(REG_ITMP2, d);
2934                                 break;
2935                         case ICMD_IFGT_ICONST:
2936                                 M_CMOVGT(REG_ITMP2, d);
2937                                 break;
2938                         case ICMD_IFLE_ICONST:
2939                                 M_CMOVLE(REG_ITMP2, d);
2940                                 break;
2941                         }
2942
2943                         store_reg_to_var_int(iptr->dst, d);
2944                         break;
2945
2946
2947                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2948                 case ICMD_LRETURN:
2949
2950                         var_to_reg_int(s1, src, REG_RESULT);
2951                         M_INTMOVE(s1, REG_RESULT);
2952                         goto nowperformreturn;
2953
2954                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2955
2956                         var_to_reg_int(s1, src, REG_RESULT);
2957                         M_INTMOVE(s1, REG_RESULT);
2958
2959 #ifdef ENABLE_VERIFIER
2960                         if (iptr->val.a) {
2961                                 PROFILE_CYCLE_STOP;
2962
2963                                 codegen_addpatchref(cd, cd->mcodeptr,
2964                                                                         PATCHER_athrow_areturn,
2965                                                                         (unresolved_class *) iptr->val.a, 0);
2966
2967                                 if (opt_showdisassemble) {
2968                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2969                                 }
2970
2971                                 PROFILE_CYCLE_START;
2972                         }
2973 #endif /* ENABLE_VERIFIER */
2974                         goto nowperformreturn;
2975
2976                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2977                 case ICMD_DRETURN:
2978
2979                         var_to_reg_flt(s1, src, REG_FRESULT);
2980                         M_FLTMOVE(s1, REG_FRESULT);
2981                         goto nowperformreturn;
2982
2983                 case ICMD_RETURN:      /* ...  ==> ...                                */
2984
2985 nowperformreturn:
2986                         {
2987                         s4 i, p;
2988
2989                         p = stackframesize;
2990
2991 #if !defined(NDEBUG)
2992                         /* generate call trace */
2993
2994                         if (opt_verbosecall) {
2995                                 x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
2996
2997                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2998                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2999
3000                                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
3001                                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
3002                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
3003                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
3004
3005                                 M_MOV_IMM(builtin_displaymethodstop, REG_ITMP1);
3006                                 M_CALL(REG_ITMP1);
3007
3008                                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
3009                                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
3010
3011                                 x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3012                         }
3013 #endif /* !defined(NDEBUG) */
3014
3015 #if defined(USE_THREADS)
3016                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
3017                                 M_ALD(rd->argintregs[0], REG_SP, rd->memuse * 8);
3018
3019                                 /* we need to save the proper return value */
3020                                 switch (iptr->opc) {
3021                                 case ICMD_IRETURN:
3022                                 case ICMD_ARETURN:
3023                                 case ICMD_LRETURN:
3024                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
3025                                         break;
3026                                 case ICMD_FRETURN:
3027                                 case ICMD_DRETURN:
3028                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
3029                                         break;
3030                                 }
3031
3032                                 M_MOV_IMM(builtin_monitorexit, REG_ITMP1);
3033                                 M_CALL(REG_ITMP1);
3034
3035                                 /* and now restore the proper return value */
3036                                 switch (iptr->opc) {
3037                                 case ICMD_IRETURN:
3038                                 case ICMD_ARETURN:
3039                                 case ICMD_LRETURN:
3040                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
3041                                         break;
3042                                 case ICMD_FRETURN:
3043                                 case ICMD_DRETURN:
3044                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
3045                                         break;
3046                                 }
3047                         }
3048 #endif
3049
3050                         /* restore saved registers */
3051
3052                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
3053                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
3054                         }
3055                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
3056                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
3057                         }
3058
3059                         /* deallocate stack */
3060
3061                         if (stackframesize)
3062                                 M_AADD_IMM(stackframesize * 8, REG_SP);
3063
3064                         /* generate method profiling code */
3065
3066                         PROFILE_CYCLE_STOP;
3067
3068                         M_RET;
3069                         }
3070                         break;
3071
3072
3073                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
3074                         {
3075                                 s4 i, l, *s4ptr;
3076                                 void **tptr;
3077
3078                                 tptr = (void **) iptr->target;
3079
3080                                 s4ptr = iptr->val.a;
3081                                 l = s4ptr[1];                          /* low     */
3082                                 i = s4ptr[2];                          /* high    */
3083
3084                                 var_to_reg_int(s1, src, REG_ITMP1);
3085                                 M_INTMOVE(s1, REG_ITMP1);
3086                                 if (l != 0) {
3087                                         x86_64_alul_imm_reg(cd, X86_64_SUB, l, REG_ITMP1);
3088                                 }
3089                                 i = i - l + 1;
3090
3091                 /* range check */
3092                                 x86_64_alul_imm_reg(cd, X86_64_CMP, i - 1, REG_ITMP1);
3093                                 x86_64_jcc(cd, X86_64_CC_A, 0);
3094
3095                 /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[0]), cd->mcodeptr); */
3096                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3097
3098                                 /* build jump table top down and use address of lowest entry */
3099
3100                 /* s4ptr += 3 + i; */
3101                                 tptr += i;
3102
3103                                 while (--i >= 0) {
3104                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
3105                                         --tptr;
3106                                 }
3107
3108                                 /* length of dataseg after last dseg_addtarget is used by load */
3109
3110                                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2);
3111                                 dseg_adddata(cd, cd->mcodeptr);
3112                                 x86_64_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
3113                                 x86_64_jmp_reg(cd, REG_ITMP1);
3114                         }
3115                         break;
3116
3117
3118                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
3119                         {
3120                                 s4 i, l, val, *s4ptr;
3121                                 void **tptr;
3122
3123                                 tptr = (void **) iptr->target;
3124
3125                                 s4ptr = iptr->val.a;
3126                                 l = s4ptr[0];                          /* default  */
3127                                 i = s4ptr[1];                          /* count    */
3128
3129                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
3130                                 var_to_reg_int(s1, src, REG_ITMP1);    /* reg compare should always be faster */
3131                                 while (--i >= 0) {
3132                                         s4ptr += 2;
3133                                         ++tptr;
3134
3135                                         val = s4ptr[0];
3136                                         x86_64_alul_imm_reg(cd, X86_64_CMP, val, s1);
3137                                         x86_64_jcc(cd, X86_64_CC_E, 0);
3138                                         codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3139                                 }
3140
3141                                 x86_64_jmp_imm(cd, 0);
3142
3143                                 tptr = (void **) iptr->target;
3144                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3145                         }
3146                         break;
3147
3148
3149                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
3150                                         /* op1 = arg count val.a = builtintable entry */
3151
3152                         bte = iptr->val.a;
3153                         md = bte->md;
3154                         goto gen_method;
3155
3156                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
3157                                         /* op1 = arg count, val.a = method pointer    */
3158
3159                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
3160                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
3161                 case ICMD_INVOKEINTERFACE:
3162
3163                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3164                                 md = INSTRUCTION_UNRESOLVED_METHOD(iptr)->methodref->parseddesc.md;
3165                                 lm = NULL;
3166                         }
3167                         else {
3168                                 lm = INSTRUCTION_RESOLVED_METHODINFO(iptr);
3169                                 md = lm->parseddesc;
3170                         }
3171
3172 gen_method:
3173                         s3 = md->paramcount;
3174
3175                         MCODECHECK((20 * s3) + 128);
3176
3177                         /* copy arguments to registers or stack location */
3178
3179                         for (s3 = s3 - 1; s3 >= 0; s3--, src = src->prev) {
3180                                 if (src->varkind == ARGVAR)
3181                                         continue;
3182                                 if (IS_INT_LNG_TYPE(src->type)) {
3183                                         if (!md->params[s3].inmemory) {
3184                                                 s1 = rd->argintregs[md->params[s3].regoff];
3185                                                 var_to_reg_int(d, src, s1);
3186                                                 M_INTMOVE(d, s1);
3187                                         } else {
3188                                                 var_to_reg_int(d, src, REG_ITMP1);
3189                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
3190                                         }
3191
3192                                 } else {
3193                                         if (!md->params[s3].inmemory) {
3194                                                 s1 = rd->argfltregs[md->params[s3].regoff];
3195                                                 var_to_reg_flt(d, src, s1);
3196                                                 M_FLTMOVE(d, s1);
3197                                         } else {
3198                                                 var_to_reg_flt(d, src, REG_FTMP1);
3199                                                 M_DST(d, REG_SP, md->params[s3].regoff * 8);
3200                                         }
3201                                 }
3202                         }
3203
3204                         /* generate method profiling code */
3205
3206                         PROFILE_CYCLE_STOP;
3207
3208                         switch (iptr->opc) {
3209                         case ICMD_BUILTIN:
3210                                 a = (ptrint) bte->fp;
3211                                 d = md->returntype.type;
3212
3213                                 M_MOV_IMM(a, REG_ITMP1);
3214                                 M_CALL(REG_ITMP1);
3215
3216                                 /* if op1 == true, we need to check for an exception */
3217
3218                                 if (iptr->op1 == true) {
3219                                         M_TEST(REG_RESULT);
3220                                         M_BEQ(0);
3221                                         codegen_add_fillinstacktrace_ref(cd, cd->mcodeptr);
3222                                 }
3223                                 break;
3224
3225                         case ICMD_INVOKESPECIAL:
3226                                 M_TEST(rd->argintregs[0]);
3227                                 M_BEQ(0);
3228                                 codegen_add_nullpointerexception_ref(cd, cd->mcodeptr);
3229
3230                                 /* first argument contains pointer */
3231 /*                              gen_nullptr_check(rd->argintregs[0]); */
3232
3233                                 /* access memory for hardware nullptr */
3234 /*                              x86_64_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); */
3235
3236                                 /* fall through */
3237
3238                         case ICMD_INVOKESTATIC:
3239                                 if (lm == NULL) {
3240                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3241
3242                                         codegen_addpatchref(cd, cd->mcodeptr,
3243                                                                                 PATCHER_invokestatic_special, um, 0);
3244
3245                                         if (opt_showdisassemble) {
3246                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3247                                         }
3248
3249                                         a = 0;
3250                                         d = um->methodref->parseddesc.md->returntype.type;
3251
3252                                 } else {
3253                                         a = (ptrint) lm->stubroutine;
3254                                         d = lm->parseddesc->returntype.type;
3255                                 }
3256
3257                                 M_MOV_IMM(a, REG_ITMP2);
3258                                 M_CALL(REG_ITMP2);
3259                                 break;
3260
3261                         case ICMD_INVOKEVIRTUAL:
3262                                 gen_nullptr_check(rd->argintregs[0]);
3263
3264                                 if (lm == NULL) {
3265                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3266
3267                                         codegen_addpatchref(cd, cd->mcodeptr,
3268                                                                                 PATCHER_invokevirtual, um, 0);
3269
3270                                         if (opt_showdisassemble) {
3271                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3272                                         }
3273
3274                                         s1 = 0;
3275                                         d = um->methodref->parseddesc.md->returntype.type;
3276
3277                                 } else {
3278                                         s1 = OFFSET(vftbl_t, table[0]) +
3279                                                 sizeof(methodptr) * lm->vftblindex;
3280                                         d = lm->parseddesc->returntype.type;
3281                                 }
3282
3283                                 x86_64_mov_membase_reg(cd, rd->argintregs[0],
3284                                                                            OFFSET(java_objectheader, vftbl),
3285                                                                            REG_ITMP2);
3286                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP1);
3287                                 M_CALL(REG_ITMP1);
3288                                 break;
3289
3290                         case ICMD_INVOKEINTERFACE:
3291                                 gen_nullptr_check(rd->argintregs[0]);
3292
3293                                 if (lm == NULL) {
3294                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3295
3296                                         codegen_addpatchref(cd, cd->mcodeptr,
3297                                                                                 PATCHER_invokeinterface, um, 0);
3298
3299                                         if (opt_showdisassemble) {
3300                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3301                                         }
3302
3303                                         s1 = 0;
3304                                         s2 = 0;
3305                                         d = um->methodref->parseddesc.md->returntype.type;
3306
3307                                 } else {
3308                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3309                                                 sizeof(methodptr) * lm->class->index;
3310
3311                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3312
3313                                         d = lm->parseddesc->returntype.type;
3314                                 }
3315
3316                                 M_ALD(REG_ITMP2, rd->argintregs[0],
3317                                           OFFSET(java_objectheader, vftbl));
3318                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP2);
3319                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s2, REG_ITMP1);
3320                                 M_CALL(REG_ITMP1);
3321                                 break;
3322                         }
3323
3324                         /* generate method profiling code */
3325
3326                         PROFILE_CYCLE_START;
3327
3328                         /* d contains return type */
3329
3330                         if (d != TYPE_VOID) {
3331                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3332                                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_RESULT);
3333                                         M_INTMOVE(REG_RESULT, s1);
3334                                         store_reg_to_var_int(iptr->dst, s1);
3335                                 } else {
3336                                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FRESULT);
3337                                         M_FLTMOVE(REG_FRESULT, s1);
3338                                         store_reg_to_var_flt(iptr->dst, s1);
3339                                 }
3340                         }
3341                         break;
3342
3343
3344                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3345
3346                                       /* op1:   0 == array, 1 == class                */
3347                                       /* val.a: (classinfo *) superclass              */
3348
3349                         /*  superclass is an interface:
3350                          *
3351                          *  OK if ((sub == NULL) ||
3352                          *         (sub->vftbl->interfacetablelength > super->index) &&
3353                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3354                          *
3355                          *  superclass is a class:
3356                          *
3357                          *  OK if ((sub == NULL) || (0
3358                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3359                          *         super->vftbl->diffval));
3360                          */
3361
3362                         if (iptr->op1 == 1) {
3363                                 /* object type cast-check */
3364
3365                                 classinfo *super;
3366                                 vftbl_t   *supervftbl;
3367                                 s4         superindex;
3368
3369                                 super = (classinfo *) iptr->val.a;
3370
3371                                 if (!super) {
3372                                         superindex = 0;
3373                                         supervftbl = NULL;
3374
3375                                 } else {
3376                                         superindex = super->index;
3377                                         supervftbl = super->vftbl;
3378                                 }
3379
3380 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3381                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3382 #endif
3383                                 var_to_reg_int(s1, src, REG_ITMP1);
3384
3385                                 /* calculate interface checkcast code size */
3386
3387                                 s2 = 3; /* mov_membase_reg */
3388                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3389
3390                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3391                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3392                                         3 /* test */ + 6 /* jcc */;
3393
3394                                 if (!super)
3395                                         s2 += (opt_showdisassemble ? 5 : 0);
3396
3397                                 /* calculate class checkcast code size */
3398
3399                                 s3 = 3; /* mov_membase_reg */
3400                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3401                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3402
3403 #if 0
3404                                 if (s1 != REG_ITMP1) {
3405                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3406                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3407                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3408                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3409                                         a += 3;    /* sub */
3410
3411                                 } else
3412 #endif
3413                                         {
3414                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3415                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3416                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3417                                         }
3418
3419                                 s3 += 3 /* cmp */ + 6 /* jcc */;
3420
3421                                 if (!super)
3422                                         s3 += (opt_showdisassemble ? 5 : 0);
3423
3424                                 /* if class is not resolved, check which code to call */
3425
3426                                 if (!super) {
3427                                         M_TEST(s1);
3428                                         M_BEQ(6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3429
3430                                         codegen_addpatchref(cd, cd->mcodeptr,
3431                                                                                 PATCHER_checkcast_instanceof_flags,
3432                                                                                 (constant_classref *) iptr->target, 0);
3433
3434                                         if (opt_showdisassemble) {
3435                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3436                                         }
3437
3438                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
3439                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
3440                                         M_BEQ(s2 + 5);
3441                                 }
3442
3443                                 /* interface checkcast code */
3444
3445                                 if (!super || (super->flags & ACC_INTERFACE)) {
3446                                         if (super) {
3447                                                 M_TEST(s1);
3448                                                 M_BEQ(s2);
3449                                         }
3450
3451                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3452
3453                                         if (!super) {
3454                                                 codegen_addpatchref(cd, cd->mcodeptr,
3455                                                                                         PATCHER_checkcast_instanceof_interface,
3456                                                                                         (constant_classref *) iptr->target, 0);
3457
3458                                                 if (opt_showdisassemble) {
3459                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3460                                                 }
3461                                         }
3462
3463                                         x86_64_movl_membase32_reg(cd, REG_ITMP2,
3464                                                                                           OFFSET(vftbl_t, interfacetablelength),
3465                                                                                           REG_ITMP3);
3466                                         /* XXX TWISTI: should this be int arithmetic? */
3467                                         M_LSUB_IMM32(superindex, REG_ITMP3);
3468                                         M_TEST(REG_ITMP3);
3469                                         M_BLE(0);
3470                                         codegen_add_classcastexception_ref(cd, cd->mcodeptr);
3471                                         x86_64_mov_membase32_reg(cd, REG_ITMP2,
3472                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
3473                                                                                          superindex * sizeof(methodptr*),
3474                                                                                          REG_ITMP3);
3475                                         M_TEST(REG_ITMP3);
3476                                         M_BEQ(0);
3477                                         codegen_add_classcastexception_ref(cd, cd->mcodeptr);
3478
3479                                         if (!super)
3480                                                 M_JMP_IMM(s3);
3481                                 }
3482
3483                                 /* class checkcast code */
3484
3485                                 if (!super || !(super->flags & ACC_INTERFACE)) {
3486                                         if (super) {
3487                                                 M_TEST(s1);
3488                                                 M_BEQ(s3);
3489                                         }
3490
3491                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3492
3493                                         if (!super) {
3494                                                 codegen_addpatchref(cd, cd->mcodeptr,
3495                                                                                         PATCHER_checkcast_class,
3496                                                                                         (constant_classref *) iptr->target,
3497                                                                                         0);
3498
3499                                                 if (opt_showdisassemble) {
3500                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3501                                                 }
3502                                         }
3503
3504                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3505 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3506                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3507 #endif
3508                                         x86_64_movl_membase32_reg(cd, REG_ITMP2,
3509                                                                                           OFFSET(vftbl_t, baseval),
3510                                                                                           REG_ITMP2);
3511                                         /*                                      if (s1 != REG_ITMP1) { */
3512                                         /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3513                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
3514                                         /*                                                                                              REG_ITMP1); */
3515                                         /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3516                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3517                                         /*                                                                                              REG_ITMP3); */
3518                                         /*  #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
3519                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3520                                         /*  #endif */
3521                                         /*                                              x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP1, REG_ITMP2); */
3522
3523                                         /*                                      } else { */
3524                                         x86_64_movl_membase32_reg(cd, REG_ITMP3,
3525                                                                                           OFFSET(vftbl_t, baseval),
3526                                                                                           REG_ITMP3);
3527                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3528                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3529                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3530                                         /*                                      } */
3531 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3532                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3533 #endif
3534                                         M_CMP(REG_ITMP3, REG_ITMP2);
3535                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3536                                         codegen_add_classcastexception_ref(cd, cd->mcodeptr);
3537                                 }
3538                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
3539
3540                         } else {
3541                                 /* array type cast-check */
3542
3543                                 var_to_reg_int(s1, src, REG_ITMP1);
3544                                 M_INTMOVE(s1, rd->argintregs[0]);
3545
3546                                 if (iptr->val.a == NULL) {
3547                                         codegen_addpatchref(cd, cd->mcodeptr,
3548                                                                                 PATCHER_builtin_arraycheckcast,
3549                                                                                 (constant_classref *) iptr->target, 0);
3550
3551                                         if (opt_showdisassemble) {
3552                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3553                                         }
3554                                 }
3555
3556                                 M_MOV_IMM(iptr->val.a, rd->argintregs[1]);
3557                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
3558                                 M_CALL(REG_ITMP1);
3559                                 M_TEST(REG_RESULT);
3560                                 M_BEQ(0);
3561                                 codegen_add_classcastexception_ref(cd, cd->mcodeptr);
3562
3563                                 var_to_reg_int(s1, src, REG_ITMP1);
3564                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
3565                         }
3566                         M_INTMOVE(s1, d);
3567                         store_reg_to_var_int(iptr->dst, d);
3568                         break;
3569
3570                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3571
3572                                       /* op1:   0 == array, 1 == class                */
3573                                       /* val.a: (classinfo *) superclass              */
3574
3575                         /*  superclass is an interface:
3576                          *
3577                          *  return (sub != NULL) &&
3578                          *         (sub->vftbl->interfacetablelength > super->index) &&
3579                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3580                          *
3581                          *  superclass is a class:
3582                          *
3583                          *  return ((sub != NULL) && (0
3584                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3585                          *          super->vftbl->diffvall));
3586                          */
3587
3588                         {
3589                         classinfo *super;
3590                         vftbl_t   *supervftbl;
3591                         s4         superindex;
3592
3593                         super = (classinfo *) iptr->val.a;
3594
3595                         if (!super) {
3596                                 superindex = 0;
3597                                 supervftbl = NULL;
3598
3599                         } else {
3600                                 superindex = super->index;
3601                                 supervftbl = super->vftbl;
3602                         }
3603
3604 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3605             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3606 #endif
3607
3608                         var_to_reg_int(s1, src, REG_ITMP1);
3609                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
3610                         if (s1 == d) {
3611                                 M_INTMOVE(s1, REG_ITMP1);
3612                                 s1 = REG_ITMP1;
3613                         }
3614
3615                         /* calculate interface instanceof code size */
3616
3617                         s2 = 3; /* mov_membase_reg */
3618                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3619                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3620                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3621                                 3 /* test */ + 4 /* setcc */;
3622
3623                         if (!super)
3624                                 s2 += (opt_showdisassemble ? 5 : 0);
3625
3626                         /* calculate class instanceof code size */
3627
3628                         s3 = 3; /* mov_membase_reg */
3629                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3630                         s3 += 10; /* mov_imm_reg */
3631                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3632                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3633                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3634                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3635                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3636                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3637                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3638
3639                         if (!super)
3640                                 s3 += (opt_showdisassemble ? 5 : 0);
3641
3642                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3643
3644                         /* if class is not resolved, check which code to call */
3645
3646                         if (!super) {
3647                                 x86_64_test_reg_reg(cd, s1, s1);
3648                                 x86_64_jcc(cd, X86_64_CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3649                                                                                          7 + 6 + s2 + 5 + s3));
3650
3651                                 codegen_addpatchref(cd, cd->mcodeptr,
3652                                                                         PATCHER_checkcast_instanceof_flags,
3653                                                                         (constant_classref *) iptr->target, 0);
3654
3655                                 if (opt_showdisassemble) {
3656                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3657                                 }
3658
3659                                 x86_64_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3660                                 x86_64_alul_imm_reg(cd, X86_64_AND, ACC_INTERFACE, REG_ITMP3);
3661                                 x86_64_jcc(cd, X86_64_CC_Z, s2 + 5);
3662                         }
3663
3664                         /* interface instanceof code */
3665
3666                         if (!super || (super->flags & ACC_INTERFACE)) {
3667                                 if (super) {
3668                                         x86_64_test_reg_reg(cd, s1, s1);
3669                                         x86_64_jcc(cd, X86_64_CC_Z, s2);
3670                                 }
3671
3672                                 x86_64_mov_membase_reg(cd, s1,
3673                                                                            OFFSET(java_objectheader, vftbl),
3674                                                                            REG_ITMP1);
3675                                 if (!super) {
3676                                         codegen_addpatchref(cd, cd->mcodeptr,
3677                                                                                 PATCHER_checkcast_instanceof_interface,
3678                                                                                 (constant_classref *) iptr->target, 0);
3679
3680                                         if (opt_showdisassemble) {
3681                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3682                                         }
3683                                 }
3684
3685                                 x86_64_movl_membase32_reg(cd, REG_ITMP1,
3686                                                                                   OFFSET(vftbl_t, interfacetablelength),
3687                                                                                   REG_ITMP3);
3688                                 x86_64_alu_imm32_reg(cd, X86_64_SUB, superindex, REG_ITMP3);
3689                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3690
3691                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3692
3693                                 x86_64_jcc(cd, X86_64_CC_LE, a);
3694                                 x86_64_mov_membase32_reg(cd, REG_ITMP1,
3695                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3696                                                                                  superindex * sizeof(methodptr*),
3697                                                                                  REG_ITMP1);
3698                                 x86_64_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3699                                 x86_64_setcc_reg(cd, X86_64_CC_NE, d);
3700
3701                                 if (!super)
3702                                         x86_64_jmp_imm(cd, s3);
3703                         }
3704
3705                         /* class instanceof code */
3706
3707                         if (!super || !(super->flags & ACC_INTERFACE)) {
3708                                 if (super) {
3709                                         x86_64_test_reg_reg(cd, s1, s1);
3710                                         x86_64_jcc(cd, X86_64_CC_E, s3);
3711                                 }
3712
3713                                 x86_64_mov_membase_reg(cd, s1,
3714                                                                            OFFSET(java_objectheader, vftbl),
3715                                                                            REG_ITMP1);
3716
3717                                 if (!super) {
3718                                         codegen_addpatchref(cd, cd->mcodeptr,
3719                                                                                 PATCHER_instanceof_class,
3720                                                                                 (constant_classref *) iptr->target, 0);
3721
3722                                         if (opt_showdisassemble) {
3723                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3724                                         }
3725                                 }
3726
3727                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3728 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3729                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3730 #endif
3731                                 x86_64_movl_membase_reg(cd, REG_ITMP1,
3732                                                                                 OFFSET(vftbl_t, baseval),
3733                                                                                 REG_ITMP1);
3734                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3735                                                                                 OFFSET(vftbl_t, diffval),
3736                                                                                 REG_ITMP3);
3737                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3738                                                                                 OFFSET(vftbl_t, baseval),
3739                                                                                 REG_ITMP2);
3740 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3741                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3742 #endif
3743                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
3744                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d); /* may be REG_ITMP2 */
3745                                 x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP3, REG_ITMP1);
3746                                 x86_64_setcc_reg(cd, X86_64_CC_BE, d);
3747                         }
3748                         store_reg_to_var_int(iptr->dst, d);
3749                         }
3750                         break;
3751
3752                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3753                                          /* op1 = dimension, val.a = class            */
3754
3755                         /* check for negative sizes and copy sizes to stack if necessary  */
3756
3757                         MCODECHECK((10 * 4 * iptr->op1) + 5 + 10 * 8);
3758
3759                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3760                                 /* copy SAVEDVAR sizes to stack */
3761
3762                                 if (src->varkind != ARGVAR) {
3763                                         var_to_reg_int(s2, src, REG_ITMP1);
3764                                         M_LST(s2, REG_SP, s1 * 8);
3765                                 }
3766                         }
3767
3768                         /* is a patcher function set? */
3769
3770                         if (iptr->val.a == NULL) {
3771                                 codegen_addpatchref(cd, cd->mcodeptr,
3772                                                                         PATCHER_builtin_multianewarray,
3773                                                                         (constant_classref *) iptr->target, 0);
3774
3775                                 if (opt_showdisassemble) {
3776                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3777                                 }
3778
3779                                 a = 0;
3780
3781                         } else {
3782                                 a = (ptrint) iptr->val.a;
3783                         }
3784
3785                         /* a0 = dimension count */
3786
3787                         M_MOV_IMM(iptr->op1, rd->argintregs[0]);
3788
3789                         /* a1 = arrayvftbl */
3790
3791                         M_MOV_IMM(iptr->val.a, rd->argintregs[1]);
3792
3793                         /* a2 = pointer to dimensions = stack pointer */
3794
3795                         M_MOV(REG_SP, rd->argintregs[2]);
3796
3797                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3798                         M_CALL(REG_ITMP1);
3799
3800                         /* check for exception before result assignment */
3801
3802                         M_TEST(REG_RESULT);
3803                         M_BEQ(0);
3804                         codegen_add_fillinstacktrace_ref(cd, cd->mcodeptr);
3805
3806                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_RESULT);
3807                         M_INTMOVE(REG_RESULT, s1);
3808                         store_reg_to_var_int(iptr->dst, s1);
3809                         break;
3810
3811                 default:
3812                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3813                         return false;
3814         } /* switch */
3815
3816         } /* for instruction */
3817
3818         /* copy values to interface registers */
3819
3820         src = bptr->outstack;
3821         len = bptr->outdepth;
3822         MCODECHECK(512);
3823 #if defined(ENABLE_LSRA)
3824         if (!opt_lsra)
3825 #endif
3826         while (src) {
3827                 len--;
3828                 if ((src->varkind != STACKVAR)) {
3829                         s2 = src->type;
3830                         if (IS_FLT_DBL_TYPE(s2)) {
3831                                 var_to_reg_flt(s1, src, REG_FTMP1);
3832                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3833                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3834
3835                                 } else {
3836                                         x86_64_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3837                                 }
3838
3839                         } else {
3840                                 var_to_reg_int(s1, src, REG_ITMP1);
3841                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3842                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3843
3844                                 } else {
3845                                         x86_64_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3846                                 }
3847                         }
3848                 }
3849                 src = src->prev;
3850         }
3851
3852         /* At the end of a basic block we may have to append some nops,
3853            because the patcher stub calling code might be longer than the
3854            actual instruction. So codepatching does not change the
3855            following block unintentionally. */
3856
3857         if (cd->mcodeptr < cd->lastmcodeptr) {
3858                 while (cd->mcodeptr < cd->lastmcodeptr) {
3859                         M_NOP;
3860                 }
3861         }
3862
3863         } /* if (bptr -> flags >= BBREACHED) */
3864         } /* for basic block */
3865
3866         dseg_createlinenumbertable(cd);
3867
3868
3869         /* generate exception and patcher stubs */
3870
3871         {
3872                 exceptionref *eref;
3873                 patchref     *pref;
3874                 ptrint        mcode;
3875                 u1           *savedmcodeptr;
3876                 u1           *tmpmcodeptr;
3877
3878                 savedmcodeptr = NULL;
3879
3880                 /* generate exception stubs */
3881
3882                 for (eref = cd->exceptionrefs; eref != NULL; eref = eref->next) {
3883                         gen_resolvebranch(cd->mcodebase + eref->branchpos,
3884                                                           eref->branchpos,
3885                                                           cd->mcodeptr - cd->mcodebase);
3886
3887                         MCODECHECK(512);
3888
3889                         /* Check if the exception is an
3890                            ArrayIndexOutOfBoundsException.  If so, move index register
3891                            into a4. */
3892
3893                         if (eref->reg != -1)
3894                                 M_MOV(eref->reg, rd->argintregs[4]);
3895
3896                         /* calcuate exception address */
3897
3898                         M_MOV_IMM(0, rd->argintregs[3]);
3899                         dseg_adddata(cd, cd->mcodeptr);
3900                         M_AADD_IMM32(eref->branchpos - 6, rd->argintregs[3]);
3901
3902                         /* move function to call into REG_ITMP3 */
3903
3904                         M_MOV_IMM(eref->function, REG_ITMP3);
3905
3906                         if (savedmcodeptr != NULL) {
3907                                 M_JMP_IMM(savedmcodeptr - cd->mcodeptr - 5);
3908
3909                         } else {
3910                                 savedmcodeptr = cd->mcodeptr;
3911
3912                                 x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3913                                 M_MOV(REG_SP, rd->argintregs[1]);
3914                                 M_ALD(rd->argintregs[2], REG_SP, stackframesize * 8);
3915
3916                                 M_ASUB_IMM(2 * 8, REG_SP);
3917                                 M_AST(rd->argintregs[3], REG_SP, 0 * 8);         /* store XPC */
3918
3919                                 M_CALL(REG_ITMP3);
3920
3921                                 M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3922                                 M_AADD_IMM(2 * 8, REG_SP);
3923
3924                                 M_MOV_IMM(asm_handle_exception, REG_ITMP3);
3925                                 M_JMP(REG_ITMP3);
3926                         }
3927                 }
3928
3929
3930                 /* generate code patching stub call code */
3931
3932                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
3933                         /* check size of code segment */
3934
3935                         MCODECHECK(512);
3936
3937                         /* Get machine code which is patched back in later. A
3938                            `call rel32' is 5 bytes long (but read 8 bytes). */
3939
3940                         savedmcodeptr = cd->mcodebase + pref->branchpos;
3941                         mcode = *((ptrint *) savedmcodeptr);
3942
3943                         /* patch in `call rel32' to call the following code */
3944
3945                         tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr          */
3946                         cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position */
3947
3948                         M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
3949
3950                         cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr   */
3951
3952                         /* move pointer to java_objectheader onto stack */
3953
3954 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3955                         /* create a virtual java_objectheader */
3956
3957                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
3958                         a = dseg_addaddress(cd, NULL);                      /* vftbl      */
3959
3960                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP3);
3961                         M_PUSH(REG_ITMP3);
3962 #else
3963                         M_PUSH_IMM(0);
3964 #endif
3965
3966                         /* move machine code bytes and classinfo pointer into registers */
3967
3968                         M_MOV_IMM(mcode, REG_ITMP3);
3969                         M_PUSH(REG_ITMP3);
3970                         M_MOV_IMM(pref->ref, REG_ITMP3);
3971                         M_PUSH(REG_ITMP3);
3972                         M_MOV_IMM(pref->disp, REG_ITMP3);
3973                         M_PUSH(REG_ITMP3);
3974
3975                         M_MOV_IMM(pref->patcher, REG_ITMP3);
3976                         M_PUSH(REG_ITMP3);
3977
3978                         M_MOV_IMM(asm_wrapper_patcher, REG_ITMP3);
3979                         M_JMP(REG_ITMP3);
3980                 }
3981         }
3982
3983         /* generate replacement-out stubs */
3984
3985         {
3986                 int i;
3987
3988                 replacementpoint = jd->code->rplpoints;
3989
3990                 for (i = 0; i < jd->code->rplpointcount; ++i, ++replacementpoint) {
3991                         /* check code segment size */
3992
3993                         MCODECHECK(512);
3994
3995                         /* note start of stub code */
3996
3997                         replacementpoint->outcode = (u1*) (ptrint)(cd->mcodeptr - cd->mcodebase);
3998
3999                         /* make machine code for patching */
4000
4001                         disp = (ptrint)(replacementpoint->outcode - replacementpoint->pc) - 5;
4002                         replacementpoint->mcode = 0xe9 | ((u8)disp << 8);
4003
4004                         /* push address of `rplpoint` struct */
4005
4006                         M_MOV_IMM(replacementpoint, REG_ITMP3);
4007                         M_PUSH(REG_ITMP3);
4008
4009                         /* jump to replacement function */
4010
4011                         M_MOV_IMM(asm_replacement_out, REG_ITMP3);
4012                         M_JMP(REG_ITMP3);
4013                 }
4014         }
4015
4016         codegen_finish(jd, (s4) (cd->mcodeptr - cd->mcodebase));
4017
4018         /* everything's ok */
4019
4020         return true;
4021 }
4022
4023
4024 /* createcompilerstub **********************************************************
4025
4026    Creates a stub routine which calls the compiler.
4027
4028 *******************************************************************************/
4029
4030 #define COMPILERSTUB_DATASIZE    2 * SIZEOF_VOID_P
4031 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
4032
4033 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
4034
4035
4036 u1 *createcompilerstub(methodinfo *m)
4037 {
4038         u1          *s;                     /* memory to hold the stub            */
4039         ptrint      *d;
4040         codegendata *cd;
4041         s4           dumpsize;
4042
4043         s = CNEW(u1, COMPILERSTUB_SIZE);
4044
4045         /* set data pointer and code pointer */
4046
4047         d = (ptrint *) s;
4048         s = s + COMPILERSTUB_DATASIZE;
4049
4050         /* mark start of dump memory area */
4051
4052         dumpsize = dump_size();
4053
4054         cd = DNEW(codegendata);
4055         cd->mcodeptr = s;
4056
4057         /* Store the methodinfo* in the same place as in the methodheader
4058            for compiled methods. */
4059
4060         d[0] = (ptrint) asm_call_jit_compiler;
4061         d[1] = (ptrint) m;
4062
4063         /* code for the stub */
4064
4065         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 1 * SIZEOF_VOID_P));       /* methodinfo  */
4066         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 2 * SIZEOF_VOID_P));  /* compiler pointer */
4067         M_JMP(REG_ITMP3);
4068
4069 #if defined(ENABLE_STATISTICS)
4070         if (opt_stat)
4071                 count_cstub_len += COMPILERSTUB_SIZE;
4072 #endif
4073
4074         /* release dump area */
4075
4076         dump_release(dumpsize);
4077
4078         return s;
4079 }
4080
4081
4082 /* createnativestub ************************************************************
4083
4084    Creates a stub routine which calls a native method.
4085
4086 *******************************************************************************/
4087
4088 u1 *createnativestub(functionptr f, jitdata *jd, methoddesc *nmd)
4089 {
4090         methodinfo   *m;
4091         codegendata  *cd;
4092         registerdata *rd;
4093         methoddesc   *md;
4094         s4            stackframesize;       /* size of stackframe if needed       */
4095         s4            nativeparams;
4096         s4            i, j;                 /* count variables                    */
4097         s4            t;
4098         s4            s1, s2;
4099
4100         /* get required compiler data */
4101
4102         m  = jd->m;
4103         cd = jd->cd;
4104         rd = jd->rd;
4105
4106         /* initialize variables */
4107
4108         md = m->parseddesc;
4109         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
4110
4111         /* calculate stack frame size */
4112
4113         stackframesize =
4114                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
4115                 sizeof(localref_table) / SIZEOF_VOID_P +
4116                 INT_ARG_CNT + FLT_ARG_CNT + 1 +         /* + 1 for function address   */
4117                 nmd->memuse;
4118
4119         if (!(stackframesize & 0x1))                /* keep stack 16-byte aligned */
4120                 stackframesize++;
4121
4122         /* create method header */
4123
4124         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
4125         (void) dseg_adds4(cd, stackframesize * 8);              /* FrameSize      */
4126         (void) dseg_adds4(cd, 0);                               /* IsSync         */
4127         (void) dseg_adds4(cd, 0);                               /* IsLeaf         */
4128         (void) dseg_adds4(cd, 0);                               /* IntSave        */
4129         (void) dseg_adds4(cd, 0);                               /* FltSave        */
4130         (void) dseg_addlinenumbertablesize(cd);
4131         (void) dseg_adds4(cd, 0);                               /* ExTableSize    */
4132
4133         /* initialize mcode variables */
4134
4135         cd->mcodeptr = (u1 *) cd->mcodebase;
4136         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
4137
4138         /* generate native method profiling code */
4139
4140         if (opt_prof) {
4141                 /* count frequency */
4142
4143                 M_MOV_IMM(m, REG_ITMP2);
4144                 M_IINC_MEMBASE(REG_ITMP2, OFFSET(methodinfo, frequency));
4145         }
4146
4147         /* generate stub code */
4148
4149         M_ASUB_IMM(stackframesize * 8, REG_SP);
4150
4151 #if !defined(NDEBUG)
4152         /* generate call trace */
4153
4154         if (opt_verbosecall) {
4155                 /* save integer and float argument registers */
4156
4157                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4158                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4159                                 M_LST(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4160
4161                 for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4162                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4163                                 M_DST(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4164
4165                 /* show integer hex code for float arguments */
4166
4167                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++) {
4168                         /* if the paramtype is a float, we have to right shift all
4169                            following integer registers */
4170
4171                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type)) {
4172                                 for (s1 = INT_ARG_CNT - 2; s1 >= i; s1--)
4173                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
4174
4175                                 x86_64_movd_freg_reg(cd, rd->argfltregs[j], rd->argintregs[i]);
4176                                 j++;
4177                         }
4178                 }
4179
4180                 M_MOV_IMM(m, REG_ITMP1);
4181                 M_AST(REG_ITMP1, REG_SP, 0 * 8);
4182                 M_MOV_IMM(builtin_trace_args, REG_ITMP1);
4183                 M_CALL(REG_ITMP1);
4184
4185                 /* restore integer and float argument registers */
4186
4187                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4188                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4189                                 M_LLD(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4190
4191                 for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4192                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4193                                 M_DLD(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4194         }
4195 #endif /* !defined(NDEBUG) */
4196
4197         /* get function address (this must happen before the stackframeinfo) */
4198
4199 #if !defined(WITH_STATIC_CLASSPATH)
4200         if (f == NULL) {
4201                 codegen_addpatchref(cd, cd->mcodeptr, PATCHER_resolve_native, m, 0);
4202
4203                 if (opt_showdisassemble) {
4204                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
4205                 }
4206         }
4207 #endif
4208
4209         M_MOV_IMM(f, REG_ITMP3);
4210
4211
4212         /* save integer and float argument registers */
4213
4214         for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4215                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4216                         M_LST(rd->argintregs[j++], REG_SP, i * 8);
4217
4218         for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4219                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4220                         M_DST(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4221
4222         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4223
4224         /* create dynamic stack info */
4225
4226         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4227         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[1]);
4228         M_ALEA(REG_SP, stackframesize * 8 + SIZEOF_VOID_P, rd->argintregs[2]);
4229         M_ALD(rd->argintregs[3], REG_SP, stackframesize * 8);
4230         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
4231         M_CALL(REG_ITMP1);
4232
4233         /* restore integer and float argument registers */
4234
4235         for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4236                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4237                         M_LLD(rd->argintregs[j++], REG_SP, i * 8);
4238
4239         for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4240                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4241                         M_DLD(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4242
4243         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4244
4245
4246         /* copy or spill arguments to new locations */
4247
4248         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
4249                 t = md->paramtypes[i].type;
4250
4251                 if (IS_INT_LNG_TYPE(t)) {
4252                         if (!md->params[i].inmemory) {
4253                                 s1 = rd->argintregs[md->params[i].regoff];
4254
4255                                 if (!nmd->params[j].inmemory) {
4256                                         s2 = rd->argintregs[nmd->params[j].regoff];
4257                                         M_INTMOVE(s1, s2);
4258
4259                                 } else {
4260                                         s2 = nmd->params[j].regoff;
4261                                         M_LST(s1, REG_SP, s2 * 8);
4262                                 }
4263
4264                         } else {
4265                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4266                                 s2 = nmd->params[j].regoff;
4267                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
4268                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
4269                         }
4270
4271                 } else {
4272                         /* We only copy spilled float arguments, as the float argument    */
4273                         /* registers keep unchanged.                                      */
4274
4275                         if (md->params[i].inmemory) {
4276                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4277                                 s2 = nmd->params[j].regoff;
4278                                 M_DLD(REG_FTMP1, REG_SP, s1 * 8);
4279                                 M_DST(REG_FTMP1, REG_SP, s2 * 8);
4280                         }
4281                 }
4282         }
4283
4284         /* put class into second argument register */
4285
4286         if (m->flags & ACC_STATIC)
4287                 M_MOV_IMM(m->class, rd->argintregs[1]);
4288
4289         /* put env into first argument register */
4290
4291         M_MOV_IMM(_Jv_env, rd->argintregs[0]);
4292
4293         /* do the native function call */
4294
4295         M_CALL(REG_ITMP3);
4296
4297         /* save return value */
4298
4299         if (md->returntype.type != TYPE_VOID) {
4300                 if (IS_INT_LNG_TYPE(md->returntype.type))
4301                         M_LST(REG_RESULT, REG_SP, 0 * 8);
4302                 else
4303                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
4304         }
4305
4306         /* remove native stackframe info */
4307
4308         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4309         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
4310         M_CALL(REG_ITMP1);
4311
4312 #if !defined(NDEBUG)
4313         /* generate call trace */
4314
4315         if (opt_verbosecall) {
4316                 /* just restore the value we need, don't care about the other */
4317
4318                 if (md->returntype.type != TYPE_VOID) {
4319                         if (IS_INT_LNG_TYPE(md->returntype.type))
4320                                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
4321                         else
4322                                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4323                 }
4324
4325                 M_MOV_IMM(m, rd->argintregs[0]);
4326                 M_MOV(REG_RESULT, rd->argintregs[1]);
4327                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4328                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4329
4330                 M_MOV_IMM(builtin_displaymethodstop, REG_ITMP1);
4331                 M_CALL(REG_ITMP1);
4332         }
4333 #endif /* !defined(NDEBUG) */
4334
4335         /* check for exception */
4336
4337 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4338         M_MOV_IMM(builtin_get_exceptionptrptr, REG_ITMP3);
4339         M_CALL(REG_ITMP3);
4340 #else
4341         M_MOV_IMM(&_no_threads_exceptionptr, REG_RESULT);
4342 #endif
4343         M_ALD(REG_ITMP2, REG_RESULT, 0);
4344
4345         /* restore return value */
4346
4347         if (md->returntype.type != TYPE_VOID) {
4348                 if (IS_INT_LNG_TYPE(md->returntype.type))
4349                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
4350                 else
4351                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4352         }
4353
4354         /* test for exception */
4355
4356         M_TEST(REG_ITMP2);
4357         M_BNE(7 + 1);
4358
4359         /* remove stackframe */
4360
4361         M_AADD_IMM(stackframesize * 8, REG_SP);
4362         M_RET;
4363
4364
4365         /* handle exception */
4366
4367 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4368         M_LST(REG_ITMP2, REG_SP, 0 * 8);
4369         M_MOV_IMM(builtin_get_exceptionptrptr, REG_ITMP3);
4370         M_CALL(REG_ITMP3);
4371         M_AST_IMM32(0, REG_RESULT, 0);                 /* clear exception pointer */
4372         M_LLD(REG_ITMP1_XPTR, REG_SP, 0 * 8);
4373 #else
4374         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
4375         M_MOV_IMM(&_no_threads_exceptionptr, REG_ITMP3);
4376         M_AST_IMM32(0, REG_ITMP3, 0);                  /* clear exception pointer */
4377 #endif
4378
4379         /* remove stackframe */
4380
4381         M_AADD_IMM(stackframesize * 8, REG_SP);
4382
4383         M_LLD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
4384         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
4385
4386         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
4387         M_JMP(REG_ITMP3);
4388
4389
4390         /* process patcher calls **************************************************/
4391
4392         {
4393                 patchref *pref;
4394                 ptrint    mcode;
4395                 u1       *savedmcodeptr;
4396                 u1       *tmpmcodeptr;
4397 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4398                 s4        disp;
4399 #endif
4400
4401                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4402                         /* Get machine code which is patched back in later. A
4403                            `call rel32' is 5 bytes long (but read 8 bytes). */
4404
4405                         savedmcodeptr = cd->mcodebase + pref->branchpos;
4406                         mcode = *((ptrint *) savedmcodeptr);
4407
4408                         /* patch in `call rel32' to call the following code               */
4409
4410                         tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr          */
4411                         cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position */
4412
4413                         M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
4414
4415                         cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr   */
4416
4417                         /* move pointer to java_objectheader onto stack */
4418
4419 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4420                         /* create a virtual java_objectheader */
4421
4422                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4423                         disp = dseg_addaddress(cd, NULL);                   /* vftbl      */
4424
4425                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP3);
4426                         M_PUSH(REG_ITMP3);
4427 #else
4428                         M_PUSH_IMM(0);
4429 #endif
4430
4431                         /* move machine code bytes and classinfo pointer into registers */
4432
4433                         M_MOV_IMM(mcode, REG_ITMP3);
4434                         M_PUSH(REG_ITMP3);
4435                         M_MOV_IMM(pref->ref, REG_ITMP3);
4436                         M_PUSH(REG_ITMP3);
4437                         M_MOV_IMM(pref->disp, REG_ITMP3);
4438                         M_PUSH(REG_ITMP3);
4439
4440                         M_MOV_IMM(pref->patcher, REG_ITMP3);
4441                         M_PUSH(REG_ITMP3);
4442
4443                         M_MOV_IMM(asm_wrapper_patcher, REG_ITMP3);
4444                         M_JMP(REG_ITMP3);
4445                 }
4446         }
4447
4448         codegen_finish(jd, (s4) (cd->mcodeptr - cd->mcodebase));
4449
4450         return jd->code->entrypoint;
4451 }
4452
4453
4454 /*
4455  * These are local overrides for various environment variables in Emacs.
4456  * Please do not remove this and leave it at the end of the file, where
4457  * Emacs will automagically detect them.
4458  * ---------------------------------------------------------------------
4459  * Local variables:
4460  * mode: c
4461  * indent-tabs-mode: t
4462  * c-basic-offset: 4
4463  * tab-width: 4
4464  * End:
4465  * vim:noexpandtab:sw=4:ts=4:
4466  */