src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31             Edwin Steiner
  32
  33    $Id: codegen.c 4826 2006-04-24 16:06:16Z twisti $
  34
  35 */
  36
  37
  38 #include "config.h"
  39
  40 #include <assert.h>
  41 #include <stdio.h>
  42
  43 #include "vm/types.h"
  44
  45 #include "md.h"
  46 #include "md-abi.h"
  47
  48 #include "vm/jit/x86_64/arch.h"
  49 #include "vm/jit/x86_64/codegen.h"
  50 #include "vm/jit/x86_64/md-emit.h"
  51
  52 #include "mm/memory.h"
  53 #include "native/jni.h"
  54 #include "native/native.h"
  55 #include "vm/builtin.h"
  56 #include "vm/exceptions.h"
  57 #include "vm/global.h"
  58 #include "vm/loader.h"
  59 #include "vm/options.h"
  60 #include "vm/statistics.h"
  61 #include "vm/stringlocal.h"
  62 #include "vm/vm.h"
  63 #include "vm/jit/asmpart.h"
  64 #include "vm/jit/codegen-common.h"
  65 #include "vm/jit/dseg.h"
  66 #include "vm/jit/emit.h"
  67 #include "vm/jit/jit.h"
  68 #include "vm/jit/methodheader.h"
  69 #include "vm/jit/parse.h"
  70 #include "vm/jit/patcher.h"
  71 #include "vm/jit/reg.h"
  72 #include "vm/jit/replace.h"
  73
  74 #if defined(ENABLE_LSRA)
  75 # include "vm/jit/allocator/lsra.h"
  76 #endif
  77
  78
  79 /* codegen *********************************************************************
  80
  81    Generates machine code.
  82
  83 *******************************************************************************/
  84
  85 bool codegen(jitdata *jd)
  86 {
  87         methodinfo         *m;
  88         codegendata        *cd;
  89         registerdata       *rd;
  90         s4                  len, s1, s2, s3, d, disp;
  91         u2                  currentline;
  92         ptrint              a;
  93         s4                  stackframesize;
  94         stackptr            src;
  95         varinfo            *var;
  96         basicblock         *bptr;
  97         instruction        *iptr;
  98         exceptiontable     *ex;
  99         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 100         builtintable_entry *bte;
 101         methoddesc         *md;
 102         rplpoint           *replacementpoint;
 103
 104         /* get required compiler data */
 105
 106         m  = jd->m;
 107         cd = jd->cd;
 108         rd = jd->rd;
 109
 110         /* prevent compiler warnings */
 111
 112         d = 0;
 113         lm = NULL;
 114         bte = NULL;
 115
 116         {
 117         s4 i, p, t, l;
 118         s4 savedregs_num;
 119
 120         savedregs_num = 0;
 121
 122         /* space to save used callee saved registers */
 123
 124         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 125         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 126
 127         stackframesize = rd->memuse + savedregs_num;
 128
 129 #if defined(USE_THREADS)
 130         /* space to save argument of monitor_enter */
 131
 132         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 133                 stackframesize++;
 134 #endif
 135
 136     /* Keep stack of non-leaf functions 16-byte aligned for calls into native */
 137         /* code e.g. libc or jni (alignment problems with movaps).                */
 138
 139         if (!m->isleafmethod || opt_verbosecall)
 140                 stackframesize |= 0x1;
 141
 142         /* create method header */
 143
 144         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 145         (void) dseg_adds4(cd, stackframesize * 8);              /* FrameSize      */
 146
 147 #if defined(USE_THREADS)
 148         /* IsSync contains the offset relative to the stack pointer for the
 149            argument of monitor_exit used in the exception handler. Since the
 150            offset could be zero and give a wrong meaning of the flag it is
 151            offset by one.
 152         */
 153
 154         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 155                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);        /* IsSync         */
 156         else
 157 #endif
 158                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 159
 160         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 161         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse);  /* IntSave        */
 162         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse);  /* FltSave        */
 163
 164         (void) dseg_addlinenumbertablesize(cd);
 165
 166         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 167
 168         /* create exception table */
 169
 170         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 171                 dseg_addtarget(cd, ex->start);
 172                 dseg_addtarget(cd, ex->end);
 173                 dseg_addtarget(cd, ex->handler);
 174                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 175         }
 176
 177         /* generate method profiling code */
 178
 179         if (opt_prof) {
 180                 /* count frequency */
 181
 182                 M_MOV_IMM(m, REG_ITMP3);
 183                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(methodinfo, frequency));
 184
 185                 PROFILE_CYCLE_START;
 186         }
 187
 188         /* create stack frame (if necessary) */
 189
 190         if (stackframesize)
 191                 M_ASUB_IMM(stackframesize * 8, REG_SP);
 192
 193         /* save used callee saved registers */
 194
 195         p = stackframesize;
 196         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 197                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 198         }
 199         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 200                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 201         }
 202
 203         /* take arguments out of register or stack frame */
 204
 205         md = m->parseddesc;
 206
 207         for (p = 0, l = 0; p < md->paramcount; p++) {
 208                 t = md->paramtypes[p].type;
 209                 var = &(rd->locals[l][t]);
 210                 l++;
 211                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 212                         l++;
 213                 if (var->type < 0)
 214                         continue;
 215                 s1 = md->params[p].regoff;
 216                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 217                         s2 = rd->argintregs[s1];
 218                         if (!md->params[p].inmemory) {           /* register arguments    */
 219                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 220                                         M_INTMOVE(s2, var->regoff);
 221
 222                                 } else {                             /* reg arg -> spilled    */
 223                                     M_LST(s2, REG_SP, var->regoff * 8);
 224                                 }
 225
 226                         } else {                                 /* stack arguments       */
 227                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 228                                         /* + 8 for return address */
 229                                         M_LLD(var->regoff, REG_SP, (stackframesize + s1) * 8 + 8);
 230
 231                                 } else {                             /* stack arg -> spilled  */
 232                                         var->regoff = stackframesize + s1 + 1;
 233                                 }
 234                         }
 235
 236                 } else {                                     /* floating args         */
 237                         if (!md->params[p].inmemory) {           /* register arguments    */
 238                                 s2 = rd->argfltregs[s1];
 239                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 240                                         M_FLTMOVE(s2, var->regoff);
 241
 242                                 } else {                                         /* reg arg -> spilled    */
 243                                         M_DST(s2, REG_SP, var->regoff * 8);
 244                                 }
 245
 246                         } else {                                 /* stack arguments       */
 247                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 248                                         M_DLD(var->regoff, REG_SP, (stackframesize + s1) * 8 + 8);
 249
 250                                 } else {
 251                                         var->regoff = stackframesize + s1 + 1;
 252                                 }
 253                         }
 254                 }
 255         }  /* end for */
 256
 257         /* save monitorenter argument */
 258
 259 #if defined(USE_THREADS)
 260         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 261                 /* stack offset for monitor argument */
 262
 263                 s1 = rd->memuse;
 264
 265                 if (opt_verbosecall) {
 266                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 267
 268                         for (p = 0; p < INT_ARG_CNT; p++)
 269                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 270
 271                         for (p = 0; p < FLT_ARG_CNT; p++)
 272                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 273
 274                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 275                 }
 276
 277                 /* decide which monitor enter function to call */
 278
 279                 if (m->flags & ACC_STATIC) {
 280                         M_MOV_IMM(m->class, REG_ITMP1);
 281                         M_AST(REG_ITMP1, REG_SP, s1 * 8);
 282                         M_INTMOVE(REG_ITMP1, rd->argintregs[0]);
 283                         M_MOV_IMM(BUILTIN_staticmonitorenter, REG_ITMP1);
 284                         M_CALL(REG_ITMP1);
 285
 286                 } else {
 287                         M_TEST(rd->argintregs[0]);
 288                         M_BEQ(0);
 289                         codegen_add_nullpointerexception_ref(cd);
 290                         M_AST(rd->argintregs[0], REG_SP, s1 * 8);
 291                         M_MOV_IMM(BUILTIN_monitorenter, REG_ITMP1);
 292                         M_CALL(REG_ITMP1);
 293                 }
 294
 295                 if (opt_verbosecall) {
 296                         for (p = 0; p < INT_ARG_CNT; p++)
 297                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 298
 299                         for (p = 0; p < FLT_ARG_CNT; p++)
 300                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 301
 302                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 303                 }
 304         }
 305 #endif
 306
 307 #if !defined(NDEBUG)
 308         /* Copy argument registers to stack and call trace function with
 309            pointer to arguments on stack. */
 310
 311         if (opt_verbosecall) {
 312                 M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 313
 314                 /* save integer argument registers */
 315
 316                 for (p = 0; p < INT_ARG_CNT; p++)
 317                         M_LST(rd->argintregs[p], REG_SP, (1 + p) * 8);
 318
 319                 /* save float argument registers */
 320
 321                 for (p = 0; p < FLT_ARG_CNT; p++)
 322                         M_DST(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 323
 324                 /* save temporary registers for leaf methods */
 325
 326                 if (m->isleafmethod) {
 327                         for (p = 0; p < INT_TMP_CNT; p++)
 328                                 M_LST(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 329
 330                         for (p = 0; p < FLT_TMP_CNT; p++)
 331                                 M_DST(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 332                 }
 333
 334                 /* show integer hex code for float arguments */
 335
 336                 for (p = 0, l = 0; p < md->paramcount && p < INT_ARG_CNT; p++) {
 337                         /* If the paramtype is a float, we have to right shift all
 338                            following integer registers. */
 339
 340                         if (IS_FLT_DBL_TYPE(md->paramtypes[p].type)) {
 341                                 for (s1 = INT_ARG_CNT - 2; s1 >= p; s1--) {
 342                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
 343                                 }
 344
 345                                 x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 346                                 l++;
 347                         }
 348                 }
 349
 350                 M_MOV_IMM(m, REG_ITMP2);
 351                 M_AST(REG_ITMP2, REG_SP, 0 * 8);
 352                 M_MOV_IMM(builtin_trace_args, REG_ITMP1);
 353                 M_CALL(REG_ITMP1);
 354
 355                 /* restore integer argument registers */
 356
 357                 for (p = 0; p < INT_ARG_CNT; p++)
 358                         M_LLD(rd->argintregs[p], REG_SP, (1 + p) * 8);
 359
 360                 /* restore float argument registers */
 361
 362                 for (p = 0; p < FLT_ARG_CNT; p++)
 363                         M_DLD(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 364
 365                 /* restore temporary registers for leaf methods */
 366
 367                 if (m->isleafmethod) {
 368                         for (p = 0; p < INT_TMP_CNT; p++)
 369                                 M_LLD(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 370
 371                         for (p = 0; p < FLT_TMP_CNT; p++)
 372                                 M_DLD(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 373                 }
 374
 375                 M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 376         }
 377 #endif /* !defined(NDEBUG) */
 378
 379         }
 380
 381         /* end of header generation */
 382
 383         replacementpoint = jd->code->rplpoints;
 384
 385         /* walk through all basic blocks */
 386
 387         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 388
 389                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 390
 391                 if (bptr->flags >= BBREACHED) {
 392
 393                         /* branch resolving */
 394
 395                         branchref *bref;
 396                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 397                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 398                                                                   bref->branchpos,
 399                                                                   bptr->mpc);
 400                         }
 401
 402                 /* handle replacement points */
 403
 404                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 405                         replacementpoint->pc = (u1*)(ptrint)bptr->mpc; /* will be resolved later */
 406
 407                         replacementpoint++;
 408
 409                         assert(cd->lastmcodeptr <= cd->mcodeptr);
 410                         cd->lastmcodeptr = cd->mcodeptr + 5; /* 5 byte jmp patch */
 411                 }
 412
 413                 /* copy interface registers to their destination */
 414
 415                 src = bptr->instack;
 416                 len = bptr->indepth;
 417                 MCODECHECK(512);
 418
 419                 /* generate basicblock profiling code */
 420
 421                 if (opt_prof_bb) {
 422                         /* count frequency */
 423
 424                         M_MOV_IMM(m->bbfrequency, REG_ITMP2);
 425                         M_IINC_MEMBASE(REG_ITMP2, bptr->debug_nr * 4);
 426
 427                         /* if this is an exception handler, start profiling again */
 428
 429                         if (bptr->type == BBTYPE_EXH)
 430                                 PROFILE_CYCLE_START;
 431                 }
 432
 433 #if defined(ENABLE_LSRA)
 434                 if (opt_lsra) {
 435                         while (src != NULL) {
 436                                 len--;
 437                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 438                                         if (bptr->type == BBTYPE_SBR) {
 439 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 440                                                 if (!(src->flags & INMEMORY))
 441                                                         d= src->regoff;
 442                                                 else
 443                                                         d=REG_ITMP1;
 444                                                 x86_64_pop_reg(cd, d);
 445                                                 emit_store(jd, NULL, src, d);
 446
 447                                         } else if (bptr->type == BBTYPE_EXH) {
 448 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 449                                                 if (!(src->flags & INMEMORY))
 450                                                         d= src->regoff;
 451                                                 else
 452                                                         d=REG_ITMP1;
 453                                                 M_INTMOVE(REG_ITMP1, d);
 454                                                 emit_store(jd, NULL, src, d);
 455                                         }
 456                                 }
 457                                 src = src->prev;
 458                         }
 459
 460                 } else {
 461 #endif
 462
 463                 while (src != NULL) {
 464                         len--;
 465                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 466                                 if (bptr->type == BBTYPE_SBR) {
 467                                         d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 468                                         M_POP(d);
 469                                         emit_store(jd, NULL, src, d);
 470
 471                                 } else if (bptr->type == BBTYPE_EXH) {
 472                                         d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 473                                         M_INTMOVE(REG_ITMP1, d);
 474                                         emit_store(jd, NULL, src, d);
 475                                 }
 476
 477                         } else {
 478                                 d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 479                                 if ((src->varkind != STACKVAR)) {
 480                                         s2 = src->type;
 481                                         if (IS_FLT_DBL_TYPE(s2)) {
 482                                                 s1 = rd->interfaces[len][s2].regoff;
 483
 484                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 485                                                         M_FLTMOVE(s1, d);
 486                                                 else
 487                                                         M_DLD(d, REG_SP, s1 * 8);
 488
 489                                                 emit_store(jd, NULL, src, d);
 490
 491                                         } else {
 492                                                 s1 = rd->interfaces[len][s2].regoff;
 493
 494                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 495                                                         M_INTMOVE(s1, d);
 496                                                 else
 497                                                         M_LLD(d, REG_SP, s1 * 8);
 498
 499                                                 emit_store(jd, NULL, src, d);
 500                                         }
 501                                 }
 502                         }
 503                         src = src->prev;
 504                 }
 505 #if defined(ENABLE_LSRA)
 506                 }
 507 #endif
 508                 /* walk through all instructions */
 509
 510                 src = bptr->instack;
 511                 len = bptr->icount;
 512                 currentline = 0;
 513
 514                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 515                         if (iptr->line != currentline) {
 516                                 dseg_addlinenumber(cd, iptr->line);
 517                                 currentline = iptr->line;
 518                         }
 519
 520                         MCODECHECK(1024);                         /* 1KB should be enough */
 521
 522                         switch (iptr->opc) {
 523                         case ICMD_INLINE_START: /* internal ICMDs                         */
 524                         case ICMD_INLINE_END:
 525                                 break;
 526
 527                         case ICMD_NOP:    /* ...  ==> ...                                 */
 528                                 break;
 529
 530                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 531
 532                                 if (src->flags & INMEMORY)
 533                                         M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
 534                                 else
 535                                         M_TEST(src->regoff);
 536                                 M_BEQ(0);
 537                                 codegen_add_nullpointerexception_ref(cd);
 538                                 break;
 539
 540                 /* constant operations ************************************************/
 541
 542                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 543                                       /* op1 = 0, val.i = constant                    */
 544
 545                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 546                         ICONST(d, iptr->val.i);
 547                         emit_store(jd, iptr, iptr->dst, d);
 548                         break;
 549
 550                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 551                                       /* op1 = 0, val.l = constant                    */
 552
 553                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 554                         LCONST(d, iptr->val.l);
 555                         emit_store(jd, iptr, iptr->dst, d);
 556                         break;
 557
 558                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 559                                       /* op1 = 0, val.f = constant                    */
 560
 561                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 562                         disp = dseg_addfloat(cd, iptr->val.f);
 563                         x86_64_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + disp, d);
 564                         emit_store(jd, iptr, iptr->dst, d);
 565                         break;
 566
 567                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 568                                       /* op1 = 0, val.d = constant                    */
 569
 570                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 571                         disp = dseg_adddouble(cd, iptr->val.d);
 572                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, d);
 573                         emit_store(jd, iptr, iptr->dst, d);
 574                         break;
 575
 576                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 577                                       /* op1 = 0, val.a = constant                    */
 578
 579                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 580
 581                         if ((iptr->target != NULL) && (iptr->val.a == NULL)) {
 582 /*                              PROFILE_CYCLE_STOP; */
 583
 584                                 codegen_addpatchref(cd, PATCHER_aconst,
 585                                                                         (unresolved_class *) iptr->target, 0);
 586
 587                                 if (opt_showdisassemble) {
 588                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
 589                                 }
 590
 591 /*                              PROFILE_CYCLE_START; */
 592
 593                                 M_MOV_IMM(iptr->val.a, d);
 594
 595                         } else {
 596                                 if (iptr->val.a == 0)
 597                                         M_CLR(d);
 598                                 else
 599                                         M_MOV_IMM(iptr->val.a, d);
 600                         }
 601                         emit_store(jd, iptr, iptr->dst, d);
 602                         break;
 603
 604
 605                 /* load/store operations **********************************************/
 606
 607                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 608                                       /* op1 = local variable                         */
 609
 610                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 611                         if ((iptr->dst->varkind == LOCALVAR) &&
 612                                 (iptr->dst->varnum == iptr->op1))
 613                                 break;
 614                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 615                         if (var->flags & INMEMORY)
 616                                 M_ILD(d, REG_SP, var->regoff * 8);
 617                         else
 618                                 M_INTMOVE(var->regoff, d);
 619                         emit_store(jd, iptr, iptr->dst, d);
 620                         break;
 621
 622                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 623                 case ICMD_ALOAD:      /* op1 = local variable                         */
 624
 625                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
 626                         if ((iptr->dst->varkind == LOCALVAR) &&
 627                                 (iptr->dst->varnum == iptr->op1))
 628                                 break;
 629                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 630                         if (var->flags & INMEMORY)
 631                                 M_LLD(d, REG_SP, var->regoff * 8);
 632                         else
 633                                 M_INTMOVE(var->regoff, d);
 634                         emit_store(jd, iptr, iptr->dst, d);
 635                         break;
 636
 637                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 638                 case ICMD_DLOAD:      /* op1 = local variable                         */
 639
 640                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
 641                         if ((iptr->dst->varkind == LOCALVAR) &&
 642                             (iptr->dst->varnum == iptr->op1))
 643                                 break;
 644                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 645                         if (var->flags & INMEMORY)
 646                                 M_DLD(d, REG_SP, var->regoff * 8);
 647                         else
 648                                 M_FLTMOVE(var->regoff, d);
 649                         emit_store(jd, iptr, iptr->dst, d);
 650                         break;
 651
 652                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 653                                       /* op1 = local variable                         */
 654
 655                         if ((src->varkind == LOCALVAR) && (src->varnum == iptr->op1))
 656                                 break;
 657                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 658                         if (var->flags & INMEMORY) {
 659                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 660                                 M_IST(s1, REG_SP, var->regoff * 8);
 661                         } else {
 662                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 663                                 M_INTMOVE(s1, var->regoff);
 664                         }
 665                         break;
 666
 667                 case ICMD_LSTORE:     /* ..., value  ==> ...                          */
 668                 case ICMD_ASTORE:     /* op1 = local variable                         */
 669
 670                         if ((src->varkind == LOCALVAR) && (src->varnum == iptr->op1))
 671                                 break;
 672                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 673                         if (var->flags & INMEMORY) {
 674                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 675                                 M_LST(s1, REG_SP, var->regoff * 8);
 676                         } else {
 677                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 678                                 M_INTMOVE(s1, var->regoff);
 679                         }
 680                         break;
 681
 682                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 683                 case ICMD_DSTORE:     /* op1 = local variable                         */
 684
 685                         if ((src->varkind == LOCALVAR) &&
 686                             (src->varnum == iptr->op1)) {
 687                                 break;
 688                         }
 689                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 690                         if (var->flags & INMEMORY) {
 691                                 s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
 692                                 M_DST(s1, REG_SP, var->regoff * 8);
 693                         } else {
 694                                 s1 = emit_load_s1(jd, iptr, src, var->regoff);
 695                                 M_FLTMOVE(s1, var->regoff);
 696                         }
 697                         break;
 698
 699
 700                 /* pop/dup/swap operations ********************************************/
 701
 702                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 703
 704                 case ICMD_POP:        /* ..., value  ==> ...                          */
 705                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 706                         break;
 707
 708                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 709                         M_COPY(src, iptr->dst);
 710                         break;
 711
 712                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 713
 714                         M_COPY(src,       iptr->dst);
 715                         M_COPY(src->prev, iptr->dst->prev);
 716                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 717                         break;
 718
 719                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 720
 721                         M_COPY(src,             iptr->dst);
 722                         M_COPY(src->prev,       iptr->dst->prev);
 723                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 724                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 725                         break;
 726
 727                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 728
 729                         M_COPY(src,       iptr->dst);
 730                         M_COPY(src->prev, iptr->dst->prev);
 731                         break;
 732
 733                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 734
 735                         M_COPY(src,             iptr->dst);
 736                         M_COPY(src->prev,       iptr->dst->prev);
 737                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 738                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 739                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 740                         break;
 741
 742                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 743
 744                         M_COPY(src,                   iptr->dst);
 745                         M_COPY(src->prev,             iptr->dst->prev);
 746                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 747                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 748                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 749                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 750                         break;
 751
 752                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 753
 754                         M_COPY(src,       iptr->dst->prev);
 755                         M_COPY(src->prev, iptr->dst);
 756                         break;
 757
 758
 759                 /* integer operations *************************************************/
 760
 761                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 762
 763                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 764                         if (iptr->dst->flags & INMEMORY) {
 765                                 if (src->flags & INMEMORY) {
 766                                         if (src->regoff == iptr->dst->regoff)
 767                                                 M_INEG_MEMBASE(REG_SP, iptr->dst->regoff * 8);
 768                                         else {
 769                                                 M_ILD(REG_ITMP1, REG_SP, src->regoff * 8);
 770                                                 M_INEG(REG_ITMP1);
 771                                                 M_IST(REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 772                                         }
 773
 774                                 } else {
 775                                         M_IST(src->regoff, REG_SP, iptr->dst->regoff * 8);
 776                                         M_INEG_MEMBASE(REG_SP, iptr->dst->regoff * 8);
 777                                 }
 778
 779                         } else {
 780                                 if (src->flags & INMEMORY) {
 781                                         M_ILD(iptr->dst->regoff, REG_SP, src->regoff * 8);
 782                                         M_INEG(iptr->dst->regoff);
 783
 784                                 } else {
 785                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 786                                         M_INEG(iptr->dst->regoff);
 787                                 }
 788                         }
 789                         break;
 790
 791                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 792
 793                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 794                         if (iptr->dst->flags & INMEMORY) {
 795                                 if (src->flags & INMEMORY) {
 796                                         if (src->regoff == iptr->dst->regoff)
 797                                                 M_LNEG_MEMBASE(REG_SP, iptr->dst->regoff * 8);
 798                                         else {
 799                                                 M_LLD(REG_ITMP1, REG_SP, src->regoff * 8);
 800                                                 M_LNEG(REG_ITMP1);
 801                                                 M_LST(REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 802                                         }
 803
 804                                 } else {
 805                                         M_LST(src->regoff, REG_SP, iptr->dst->regoff * 8);
 806                                         M_LNEG_MEMBASE(REG_SP, iptr->dst->regoff * 8);
 807                                 }
 808
 809                         } else {
 810                                 if (src->flags & INMEMORY) {
 811                                         M_LLD(iptr->dst->regoff, REG_SP, src->regoff * 8);
 812                                         M_LNEG(iptr->dst->regoff);
 813
 814                                 } else {
 815                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 816                                         M_LNEG(iptr->dst->regoff);
 817                                 }
 818                         }
 819                         break;
 820
 821                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 822
 823                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 824                         if (src->flags & INMEMORY) {
 825                                 x86_64_movslq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 826
 827                         } else {
 828                                 x86_64_movslq_reg_reg(cd, src->regoff, d);
 829                         }
 830                         emit_store(jd, iptr, iptr->dst, d);
 831                         break;
 832
 833                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 834
 835                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
 836                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 837                         M_INTMOVE(s1, d);
 838                         emit_store(jd, iptr, iptr->dst, d);
 839                         break;
 840
 841                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 842
 843                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 844                         if (src->flags & INMEMORY) {
 845                                 x86_64_movsbq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 846
 847                         } else {
 848                                 x86_64_movsbq_reg_reg(cd, src->regoff, d);
 849                         }
 850                         emit_store(jd, iptr, iptr->dst, d);
 851                         break;
 852
 853                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 854
 855                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 856                         if (src->flags & INMEMORY) {
 857                                 x86_64_movzwq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 858
 859                         } else {
 860                                 x86_64_movzwq_reg_reg(cd, src->regoff, d);
 861                         }
 862                         emit_store(jd, iptr, iptr->dst, d);
 863                         break;
 864
 865                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 866
 867                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
 868                         if (src->flags & INMEMORY) {
 869                                 x86_64_movswq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 870
 871                         } else {
 872                                 x86_64_movswq_reg_reg(cd, src->regoff, d);
 873                         }
 874                         emit_store(jd, iptr, iptr->dst, d);
 875                         break;
 876
 877
 878                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 879
 880                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 881                         x86_64_emit_ialu(cd, X86_64_ADD, src, iptr);
 882                         break;
 883
 884                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 885                                       /* val.i = constant                             */
 886
 887                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 888                         x86_64_emit_ialuconst(cd, X86_64_ADD, src, iptr);
 889                         break;
 890
 891                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 892
 893                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 894                         x86_64_emit_lalu(cd, X86_64_ADD, src, iptr);
 895                         break;
 896
 897                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 898                                       /* val.l = constant                             */
 899
 900                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 901                         x86_64_emit_laluconst(cd, X86_64_ADD, src, iptr);
 902                         break;
 903
 904                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 905
 906                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 907                         if (iptr->dst->flags & INMEMORY) {
 908                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 909                                         if (src->prev->regoff == iptr->dst->regoff) {
 910                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 911                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 912
 913                                         } else {
 914                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 915                                                 x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 916                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 917                                         }
 918
 919                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 920                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 921                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 922                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 923
 924                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 925                                         if (src->prev->regoff == iptr->dst->regoff) {
 926                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 927
 928                                         } else {
 929                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 930                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 931                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 932                                         }
 933
 934                                 } else {
 935                                         x86_64_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 936                                         x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 937                                 }
 938
 939                         } else {
 940                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 941                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 942                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 943
 944                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 945                                         M_INTMOVE(src->prev->regoff, d);
 946                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 947
 948                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 949                                         /* workaround for reg alloc */
 950                                         if (src->regoff == iptr->dst->regoff) {
 951                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 952                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 953                                                 M_INTMOVE(REG_ITMP1, d);
 954
 955                                         } else {
 956                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 957                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 958                                         }
 959
 960                                 } else {
 961                                         /* workaround for reg alloc */
 962                                         if (src->regoff == iptr->dst->regoff) {
 963                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
 964                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 965                                                 M_INTMOVE(REG_ITMP1, d);
 966
 967                                         } else {
 968                                                 M_INTMOVE(src->prev->regoff, d);
 969                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 970                                         }
 971                                 }
 972                         }
 973                         break;
 974
 975                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 976                                       /* val.i = constant                             */
 977
 978                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 979                         x86_64_emit_ialuconst(cd, X86_64_SUB, src, iptr);
 980                         break;
 981
 982                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 983
 984                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
 985                         if (iptr->dst->flags & INMEMORY) {
 986                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 987                                         if (src->prev->regoff == iptr->dst->regoff) {
 988                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 989                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 990
 991                                         } else {
 992                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 993                                                 x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 994                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 995                                         }
 996
 997                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 998                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 999                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1000                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1001
1002                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1003                                         if (src->prev->regoff == iptr->dst->regoff) {
1004                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1005
1006                                         } else {
1007                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1008                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1009                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1010                                         }
1011
1012                                 } else {
1013                                         x86_64_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
1014                                         x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1015                                 }
1016
1017                         } else {
1018                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1019                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1020                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1021
1022                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1023                                         M_INTMOVE(src->prev->regoff, d);
1024                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1025
1026                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1027                                         /* workaround for reg alloc */
1028                                         if (src->regoff == iptr->dst->regoff) {
1029                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1030                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1031                                                 M_INTMOVE(REG_ITMP1, d);
1032
1033                                         } else {
1034                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1035                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1036                                         }
1037
1038                                 } else {
1039                                         /* workaround for reg alloc */
1040                                         if (src->regoff == iptr->dst->regoff) {
1041                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1042                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1043                                                 M_INTMOVE(REG_ITMP1, d);
1044
1045                                         } else {
1046                                                 M_INTMOVE(src->prev->regoff, d);
1047                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1048                                         }
1049                                 }
1050                         }
1051                         break;
1052
1053                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1054                                       /* val.l = constant                             */
1055
1056                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1057                         x86_64_emit_laluconst(cd, X86_64_SUB, src, iptr);
1058                         break;
1059
1060                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1061
1062                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1063                         if (iptr->dst->flags & INMEMORY) {
1064                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1065                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1066                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1067                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1068
1069                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1070                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1071                                         x86_64_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1072                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1073
1074                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1075                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1076                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1077                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1078
1079                                 } else {
1080                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1081                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1082                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1083                                 }
1084
1085                         } else {
1086                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1087                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1088                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1089
1090                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1091                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1092                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1093
1094                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1095                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1096                                         x86_64_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1097
1098                                 } else {
1099                                         if (src->regoff == iptr->dst->regoff) {
1100                                                 x86_64_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1101
1102                                         } else {
1103                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1104                                                 x86_64_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1105                                         }
1106                                 }
1107                         }
1108                         break;
1109
1110                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1111                                       /* val.i = constant                             */
1112
1113                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1114                         if (iptr->dst->flags & INMEMORY) {
1115                                 if (src->flags & INMEMORY) {
1116                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1117                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1118
1119                                 } else {
1120                                         x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1121                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1122                                 }
1123
1124                         } else {
1125                                 if (src->flags & INMEMORY) {
1126                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1127
1128                                 } else {
1129                                         if (iptr->val.i == 2) {
1130                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1131                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1132
1133                                         } else {
1134                                                 x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1135                                         }
1136                                 }
1137                         }
1138                         break;
1139
1140                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1141
1142                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1143                         if (iptr->dst->flags & INMEMORY) {
1144                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1145                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1146                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1147                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1148
1149                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1150                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1151                                         x86_64_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1152                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1153
1154                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1155                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1156                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1157                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1158
1159                                 } else {
1160                                         x86_64_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1161                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1162                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1163                                 }
1164
1165                         } else {
1166                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1167                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1168                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1169
1170                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1171                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1172                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1173
1174                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1175                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1176                                         x86_64_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1177
1178                                 } else {
1179                                         if (src->regoff == iptr->dst->regoff) {
1180                                                 x86_64_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1181
1182                                         } else {
1183                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1184                                                 x86_64_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1185                                         }
1186                                 }
1187                         }
1188                         break;
1189
1190                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1191                                       /* val.l = constant                             */
1192
1193                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1194                         if (iptr->dst->flags & INMEMORY) {
1195                                 if (src->flags & INMEMORY) {
1196                                         if (IS_IMM32(iptr->val.l)) {
1197                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1198
1199                                         } else {
1200                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1201                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1202                                         }
1203                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1204
1205                                 } else {
1206                                         if (IS_IMM32(iptr->val.l)) {
1207                                                 x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1208
1209                                         } else {
1210                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1211                                                 x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1212                                         }
1213                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1214                                 }
1215
1216                         } else {
1217                                 if (src->flags & INMEMORY) {
1218                                         if (IS_IMM32(iptr->val.l)) {
1219                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1220
1221                                         } else {
1222                                                 x86_64_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1223                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1224                                         }
1225
1226                                 } else {
1227                                         /* should match in many cases */
1228                                         if (iptr->val.l == 2) {
1229                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1230                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1231
1232                                         } else {
1233                                                 if (IS_IMM32(iptr->val.l)) {
1234                                                         x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1235
1236                                                 } else {
1237                                                         x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1238                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1239                                                         x86_64_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1240                                                 }
1241                                         }
1242                                 }
1243                         }
1244                         break;
1245
1246                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1247
1248                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1249                 if (src->prev->flags & INMEMORY) {
1250                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1251
1252                         } else {
1253                                 M_INTMOVE(src->prev->regoff, RAX);
1254                         }
1255
1256                         if (src->flags & INMEMORY) {
1257                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1258
1259                         } else {
1260                                 M_INTMOVE(src->regoff, REG_ITMP3);
1261                         }
1262                         gen_div_check(src);
1263
1264                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1265                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1266                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1267                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                /* 6 bytes */
1268
1269                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1270                         x86_64_cltd(cd);
1271                         x86_64_idivl_reg(cd, REG_ITMP3);
1272
1273                         if (iptr->dst->flags & INMEMORY) {
1274                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1275                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1276
1277                         } else {
1278                                 M_INTMOVE(RAX, iptr->dst->regoff);
1279
1280                                 if (iptr->dst->regoff != RDX) {
1281                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1282                                 }
1283                         }
1284                         break;
1285
1286                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1287                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1288                         if (src->prev->flags & INMEMORY) {
1289                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1290
1291                         } else {
1292                                 M_INTMOVE(src->prev->regoff, RAX);
1293                         }
1294
1295                         if (src->flags & INMEMORY) {
1296                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1297
1298                         } else {
1299                                 M_INTMOVE(src->regoff, REG_ITMP3);
1300                         }
1301                         gen_div_check(src);
1302
1303                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1304
1305                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1306                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1307
1308
1309                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1310                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1311                         x86_64_jcc(cd, X86_64_CC_E, 1 + 3);                    /* 6 bytes */
1312
1313                         x86_64_cltd(cd);
1314                         x86_64_idivl_reg(cd, REG_ITMP3);
1315
1316                         if (iptr->dst->flags & INMEMORY) {
1317                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1318                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1319
1320                         } else {
1321                                 M_INTMOVE(RDX, iptr->dst->regoff);
1322
1323                                 if (iptr->dst->regoff != RDX) {
1324                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1325                                 }
1326                         }
1327                         break;
1328
1329                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1330                                       /* val.i = constant                             */
1331
1332                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1333                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1334                         M_INTMOVE(s1, REG_ITMP1);
1335                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1336                         x86_64_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1337                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1338                         x86_64_shiftl_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1339                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1340                         emit_store(jd, iptr, iptr->dst, d);
1341                         break;
1342
1343                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1344                                       /* val.i = constant                             */
1345
1346                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1347                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1348                         M_INTMOVE(s1, REG_ITMP1);
1349                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1350                         x86_64_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1351                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1352                         x86_64_alul_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1353                         x86_64_alul_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1354                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1355                         emit_store(jd, iptr, iptr->dst, d);
1356                         break;
1357
1358
1359                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1360
1361                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1362
1363                 if (src->prev->flags & INMEMORY) {
1364                                 M_LLD(RAX, REG_SP, src->prev->regoff * 8);
1365
1366                         } else {
1367                                 M_INTMOVE(src->prev->regoff, RAX);
1368                         }
1369
1370                         if (src->flags & INMEMORY) {
1371                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1372
1373                         } else {
1374                                 M_INTMOVE(src->regoff, REG_ITMP3);
1375                         }
1376                         gen_div_check(src);
1377
1378                         /* check as described in jvm spec */
1379                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1380                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, RAX);
1381                         M_BNE(4 + 6);
1382                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1383                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
1384
1385                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1386                         x86_64_cqto(cd);
1387                         x86_64_idiv_reg(cd, REG_ITMP3);
1388
1389                         if (iptr->dst->flags & INMEMORY) {
1390                                 M_LST(RAX, REG_SP, iptr->dst->regoff * 8);
1391                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1392
1393                         } else {
1394                                 M_INTMOVE(RAX, iptr->dst->regoff);
1395
1396                                 if (iptr->dst->regoff != RDX) {
1397                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1398                                 }
1399                         }
1400                         break;
1401
1402                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1403
1404                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1405                         if (src->prev->flags & INMEMORY) {
1406                                 M_LLD(REG_ITMP1, REG_SP, src->prev->regoff * 8);
1407
1408                         } else {
1409                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1410                         }
1411
1412                         if (src->flags & INMEMORY) {
1413                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1414
1415                         } else {
1416                                 M_INTMOVE(src->regoff, REG_ITMP3);
1417                         }
1418                         gen_div_check(src);
1419
1420                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1421
1422                         /* check as described in jvm spec */
1423                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1424                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP1);
1425                         M_BNE(3 + 4 + 6);
1426
1427 #if 0
1428                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1429 #endif
1430                         M_XOR(RDX, RDX);                                       /* 3 bytes */
1431                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1432                         M_BEQ(2 + 3);                                          /* 6 bytes */
1433
1434                         x86_64_cqto(cd);
1435                         x86_64_idiv_reg(cd, REG_ITMP3);
1436
1437                         if (iptr->dst->flags & INMEMORY) {
1438                                 M_LST(RDX, REG_SP, iptr->dst->regoff * 8);
1439                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1440
1441                         } else {
1442                                 M_INTMOVE(RDX, iptr->dst->regoff);
1443
1444                                 if (iptr->dst->regoff != RDX) {
1445                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1446                                 }
1447                         }
1448                         break;
1449
1450                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1451                                       /* val.i = constant                             */
1452
1453                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1454                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1455                         M_INTMOVE(s1, REG_ITMP1);
1456                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1457                         x86_64_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1458                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1459                         x86_64_shift_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1460                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1461                         emit_store(jd, iptr, iptr->dst, d);
1462                         break;
1463
1464                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1465                                       /* val.l = constant                             */
1466
1467                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1468                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1469                         M_INTMOVE(s1, REG_ITMP1);
1470                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1471                         x86_64_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1472                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1473                         x86_64_alu_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1474                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1475                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1476                         emit_store(jd, iptr, iptr->dst, d);
1477                         break;
1478
1479                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1480
1481                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1482                         x86_64_emit_ishift(cd, X86_64_SHL, src, iptr);
1483                         break;
1484
1485                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1486                                       /* val.i = constant                             */
1487
1488                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1489                         x86_64_emit_ishiftconst(cd, X86_64_SHL, src, iptr);
1490                         break;
1491
1492                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1493
1494                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1495                         x86_64_emit_ishift(cd, X86_64_SAR, src, iptr);
1496                         break;
1497
1498                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1499                                       /* val.i = constant                             */
1500
1501                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1502                         x86_64_emit_ishiftconst(cd, X86_64_SAR, src, iptr);
1503                         break;
1504
1505                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1506
1507                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1508                         x86_64_emit_ishift(cd, X86_64_SHR, src, iptr);
1509                         break;
1510
1511                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1512                                       /* val.i = constant                             */
1513
1514                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1515                         x86_64_emit_ishiftconst(cd, X86_64_SHR, src, iptr);
1516                         break;
1517
1518                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1519
1520                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1521                         x86_64_emit_lshift(cd, X86_64_SHL, src, iptr);
1522                         break;
1523
1524         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1525                                           /* val.i = constant                             */
1526
1527                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1528                         x86_64_emit_lshiftconst(cd, X86_64_SHL, src, iptr);
1529                         break;
1530
1531                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1532
1533                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1534                         x86_64_emit_lshift(cd, X86_64_SAR, src, iptr);
1535                         break;
1536
1537                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1538                                       /* val.i = constant                             */
1539
1540                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1541                         x86_64_emit_lshiftconst(cd, X86_64_SAR, src, iptr);
1542                         break;
1543
1544                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1545
1546                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1547                         x86_64_emit_lshift(cd, X86_64_SHR, src, iptr);
1548                         break;
1549
1550                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1551                                       /* val.l = constant                             */
1552
1553                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1554                         x86_64_emit_lshiftconst(cd, X86_64_SHR, src, iptr);
1555                         break;
1556
1557                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1558
1559                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1560                         x86_64_emit_ialu(cd, X86_64_AND, src, iptr);
1561                         break;
1562
1563                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1564                                       /* val.i = constant                             */
1565
1566                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1567                         x86_64_emit_ialuconst(cd, X86_64_AND, src, iptr);
1568                         break;
1569
1570                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1571
1572                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1573                         x86_64_emit_lalu(cd, X86_64_AND, src, iptr);
1574                         break;
1575
1576                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1577                                       /* val.l = constant                             */
1578
1579                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1580                         x86_64_emit_laluconst(cd, X86_64_AND, src, iptr);
1581                         break;
1582
1583                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1584
1585                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1586                         x86_64_emit_ialu(cd, X86_64_OR, src, iptr);
1587                         break;
1588
1589                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1590                                       /* val.i = constant                             */
1591
1592                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1593                         x86_64_emit_ialuconst(cd, X86_64_OR, src, iptr);
1594                         break;
1595
1596                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1597
1598                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1599                         x86_64_emit_lalu(cd, X86_64_OR, src, iptr);
1600                         break;
1601
1602                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1603                                       /* val.l = constant                             */
1604
1605                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1606                         x86_64_emit_laluconst(cd, X86_64_OR, src, iptr);
1607                         break;
1608
1609                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1610
1611                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1612                         x86_64_emit_ialu(cd, X86_64_XOR, src, iptr);
1613                         break;
1614
1615                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1616                                       /* val.i = constant                             */
1617
1618                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1619                         x86_64_emit_ialuconst(cd, X86_64_XOR, src, iptr);
1620                         break;
1621
1622                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1623
1624                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1625                         x86_64_emit_lalu(cd, X86_64_XOR, src, iptr);
1626                         break;
1627
1628                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1629                                       /* val.l = constant                             */
1630
1631                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_NULL);
1632                         x86_64_emit_laluconst(cd, X86_64_XOR, src, iptr);
1633                         break;
1634
1635
1636                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1637                                       /* op1 = variable, val.i = constant             */
1638
1639                         /* using inc and dec is definitely faster than add -- tested      */
1640                         /* with sieve                                                     */
1641
1642                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1643                         d = var->regoff;
1644                         if (var->flags & INMEMORY) {
1645                                 if (iptr->val.i == 1) {
1646                                         x86_64_incl_membase(cd, REG_SP, d * 8);
1647
1648                                 } else if (iptr->val.i == -1) {
1649                                         x86_64_decl_membase(cd, REG_SP, d * 8);
1650
1651                                 } else {
1652                                         x86_64_alul_imm_membase(cd, X86_64_ADD, iptr->val.i, REG_SP, d * 8);
1653                                 }
1654
1655                         } else {
1656                                 if (iptr->val.i == 1) {
1657                                         x86_64_incl_reg(cd, d);
1658
1659                                 } else if (iptr->val.i == -1) {
1660                                         x86_64_decl_reg(cd, d);
1661
1662                                 } else {
1663                                         x86_64_alul_imm_reg(cd, X86_64_ADD, iptr->val.i, d);
1664                                 }
1665                         }
1666                         break;
1667
1668
1669                 /* floating operations ************************************************/
1670
1671                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1672
1673                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1674                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1675                         disp = dseg_adds4(cd, 0x80000000);
1676                         M_FLTMOVE(s1, d);
1677                         x86_64_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1678                         x86_64_xorps_reg_reg(cd, REG_FTMP2, d);
1679                         emit_store(jd, iptr, iptr->dst, d);
1680                         break;
1681
1682                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1683
1684                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1685                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1686                         disp = dseg_adds8(cd, 0x8000000000000000);
1687                         M_FLTMOVE(s1, d);
1688                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1689                         x86_64_xorpd_reg_reg(cd, REG_FTMP2, d);
1690                         emit_store(jd, iptr, iptr->dst, d);
1691                         break;
1692
1693                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1694
1695                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1696                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1697                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1698                         if (s1 == d) {
1699                                 x86_64_addss_reg_reg(cd, s2, d);
1700                         } else if (s2 == d) {
1701                                 x86_64_addss_reg_reg(cd, s1, d);
1702                         } else {
1703                                 M_FLTMOVE(s1, d);
1704                                 x86_64_addss_reg_reg(cd, s2, d);
1705                         }
1706                         emit_store(jd, iptr, iptr->dst, d);
1707                         break;
1708
1709                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1710
1711                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1712                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1713                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1714                         if (s1 == d) {
1715                                 x86_64_addsd_reg_reg(cd, s2, d);
1716                         } else if (s2 == d) {
1717                                 x86_64_addsd_reg_reg(cd, s1, d);
1718                         } else {
1719                                 M_FLTMOVE(s1, d);
1720                                 x86_64_addsd_reg_reg(cd, s2, d);
1721                         }
1722                         emit_store(jd, iptr, iptr->dst, d);
1723                         break;
1724
1725                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1726
1727                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1728                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1729                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1730                         if (s2 == d) {
1731                                 M_FLTMOVE(s2, REG_FTMP2);
1732                                 s2 = REG_FTMP2;
1733                         }
1734                         M_FLTMOVE(s1, d);
1735                         x86_64_subss_reg_reg(cd, s2, d);
1736                         emit_store(jd, iptr, iptr->dst, d);
1737                         break;
1738
1739                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1740
1741                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1742                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1743                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1744                         if (s2 == d) {
1745                                 M_FLTMOVE(s2, REG_FTMP2);
1746                                 s2 = REG_FTMP2;
1747                         }
1748                         M_FLTMOVE(s1, d);
1749                         x86_64_subsd_reg_reg(cd, s2, d);
1750                         emit_store(jd, iptr, iptr->dst, d);
1751                         break;
1752
1753                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1754
1755                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1756                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1757                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1758                         if (s1 == d) {
1759                                 x86_64_mulss_reg_reg(cd, s2, d);
1760                         } else if (s2 == d) {
1761                                 x86_64_mulss_reg_reg(cd, s1, d);
1762                         } else {
1763                                 M_FLTMOVE(s1, d);
1764                                 x86_64_mulss_reg_reg(cd, s2, d);
1765                         }
1766                         emit_store(jd, iptr, iptr->dst, d);
1767                         break;
1768
1769                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1770
1771                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1772                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1773                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1774                         if (s1 == d) {
1775                                 x86_64_mulsd_reg_reg(cd, s2, d);
1776                         } else if (s2 == d) {
1777                                 x86_64_mulsd_reg_reg(cd, s1, d);
1778                         } else {
1779                                 M_FLTMOVE(s1, d);
1780                                 x86_64_mulsd_reg_reg(cd, s2, d);
1781                         }
1782                         emit_store(jd, iptr, iptr->dst, d);
1783                         break;
1784
1785                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1786
1787                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1788                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1789                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1790                         if (s2 == d) {
1791                                 M_FLTMOVE(s2, REG_FTMP2);
1792                                 s2 = REG_FTMP2;
1793                         }
1794                         M_FLTMOVE(s1, d);
1795                         x86_64_divss_reg_reg(cd, s2, d);
1796                         emit_store(jd, iptr, iptr->dst, d);
1797                         break;
1798
1799                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1800
1801                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1802                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1803                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1804                         if (s2 == d) {
1805                                 M_FLTMOVE(s2, REG_FTMP2);
1806                                 s2 = REG_FTMP2;
1807                         }
1808                         M_FLTMOVE(s1, d);
1809                         x86_64_divsd_reg_reg(cd, s2, d);
1810                         emit_store(jd, iptr, iptr->dst, d);
1811                         break;
1812
1813                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1814
1815                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1816                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1817                         x86_64_cvtsi2ss_reg_reg(cd, s1, d);
1818                         emit_store(jd, iptr, iptr->dst, d);
1819                         break;
1820
1821                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1822
1823                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1824                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1825                         x86_64_cvtsi2sd_reg_reg(cd, s1, d);
1826                         emit_store(jd, iptr, iptr->dst, d);
1827                         break;
1828
1829                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1830
1831                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1832                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1833                         x86_64_cvtsi2ssq_reg_reg(cd, s1, d);
1834                         emit_store(jd, iptr, iptr->dst, d);
1835                         break;
1836
1837                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1838
1839                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1840                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
1841                         x86_64_cvtsi2sdq_reg_reg(cd, s1, d);
1842                         emit_store(jd, iptr, iptr->dst, d);
1843                         break;
1844
1845                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1846
1847                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1848                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1849                         x86_64_cvttss2si_reg_reg(cd, s1, d);
1850                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1851                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1852                         x86_64_jcc(cd, X86_64_CC_NE, a);
1853                         M_FLTMOVE(s1, REG_FTMP1);
1854                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_f2i, REG_ITMP2);
1855                         x86_64_call_reg(cd, REG_ITMP2);
1856                         M_INTMOVE(REG_RESULT, d);
1857                         emit_store(jd, iptr, iptr->dst, d);
1858                         break;
1859
1860                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1861
1862                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1863                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1864                         x86_64_cvttsd2si_reg_reg(cd, s1, d);
1865                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1866                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1867                         x86_64_jcc(cd, X86_64_CC_NE, a);
1868                         M_FLTMOVE(s1, REG_FTMP1);
1869                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_d2i, REG_ITMP2);
1870                         x86_64_call_reg(cd, REG_ITMP2);
1871                         M_INTMOVE(REG_RESULT, d);
1872                         emit_store(jd, iptr, iptr->dst, d);
1873                         break;
1874
1875                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1876
1877                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1878                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1879                         x86_64_cvttss2siq_reg_reg(cd, s1, d);
1880                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1881                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1882                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1883                         x86_64_jcc(cd, X86_64_CC_NE, a);
1884                         M_FLTMOVE(s1, REG_FTMP1);
1885                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_f2l, REG_ITMP2);
1886                         x86_64_call_reg(cd, REG_ITMP2);
1887                         M_INTMOVE(REG_RESULT, d);
1888                         emit_store(jd, iptr, iptr->dst, d);
1889                         break;
1890
1891                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1892
1893                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1894                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
1895                         x86_64_cvttsd2siq_reg_reg(cd, s1, d);
1896                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1897                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1898                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1899                         x86_64_jcc(cd, X86_64_CC_NE, a);
1900                         M_FLTMOVE(s1, REG_FTMP1);
1901                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_d2l, REG_ITMP2);
1902                         x86_64_call_reg(cd, REG_ITMP2);
1903                         M_INTMOVE(REG_RESULT, d);
1904                         emit_store(jd, iptr, iptr->dst, d);
1905                         break;
1906
1907                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1908
1909                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1910                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1911                         x86_64_cvtss2sd_reg_reg(cd, s1, d);
1912                         emit_store(jd, iptr, iptr->dst, d);
1913                         break;
1914
1915                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1916
1917                         s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
1918                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
1919                         x86_64_cvtsd2ss_reg_reg(cd, s1, d);
1920                         emit_store(jd, iptr, iptr->dst, d);
1921                         break;
1922
1923                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1924                                           /* == => 0, < => 1, > => -1 */
1925
1926                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1927                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1928                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1929                         M_CLR(d);
1930                         M_MOV_IMM(1, REG_ITMP1);
1931                         M_MOV_IMM(-1, REG_ITMP2);
1932                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1933                         M_CMOVB(REG_ITMP1, d);
1934                         M_CMOVA(REG_ITMP2, d);
1935                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1936                         emit_store(jd, iptr, iptr->dst, d);
1937                         break;
1938
1939                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1940                                           /* == => 0, < => 1, > => -1 */
1941
1942                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1943                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1944                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1945                         M_CLR(d);
1946                         M_MOV_IMM(1, REG_ITMP1);
1947                         M_MOV_IMM(-1, REG_ITMP2);
1948                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1949                         M_CMOVB(REG_ITMP1, d);
1950                         M_CMOVA(REG_ITMP2, d);
1951                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1952                         emit_store(jd, iptr, iptr->dst, d);
1953                         break;
1954
1955                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1956                                           /* == => 0, < => 1, > => -1 */
1957
1958                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1959                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1960                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1961                         M_CLR(d);
1962                         M_MOV_IMM(1, REG_ITMP1);
1963                         M_MOV_IMM(-1, REG_ITMP2);
1964                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1965                         M_CMOVB(REG_ITMP1, d);
1966                         M_CMOVA(REG_ITMP2, d);
1967                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1968                         emit_store(jd, iptr, iptr->dst, d);
1969                         break;
1970
1971                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1972                                           /* == => 0, < => 1, > => -1 */
1973
1974                         s1 = emit_load_s1(jd, iptr, src->prev, REG_FTMP1);
1975                         s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
1976                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1977                         M_CLR(d);
1978                         M_MOV_IMM(1, REG_ITMP1);
1979                         M_MOV_IMM(-1, REG_ITMP2);
1980                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1981                         M_CMOVB(REG_ITMP1, d);
1982                         M_CMOVA(REG_ITMP2, d);
1983                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1984                         emit_store(jd, iptr, iptr->dst, d);
1985                         break;
1986
1987
1988                 /* memory operations **************************************************/
1989
1990                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1991
1992                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
1993                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
1994                         gen_nullptr_check(s1);
1995                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1996                         emit_store(jd, iptr, iptr->dst, d);
1997                         break;
1998
1999                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2000
2001                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2002                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2003                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2004                         if (iptr->op1 == 0) {
2005                                 gen_nullptr_check(s1);
2006                                 gen_bound_check;
2007                         }
2008                         x86_64_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2009                         emit_store(jd, iptr, iptr->dst, d);
2010                         break;
2011
2012                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2013
2014                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2015                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2016                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2017                         if (iptr->op1 == 0) {
2018                                 gen_nullptr_check(s1);
2019                                 gen_bound_check;
2020                         }
2021                         x86_64_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2022                         emit_store(jd, iptr, iptr->dst, d);
2023                         break;
2024
2025                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2026
2027                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2028                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2029                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2030                         if (iptr->op1 == 0) {
2031                                 gen_nullptr_check(s1);
2032                                 gen_bound_check;
2033                         }
2034                         x86_64_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2035                         emit_store(jd, iptr, iptr->dst, d);
2036                         break;
2037
2038                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
2039
2040                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2041                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2042                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2043                         if (iptr->op1 == 0) {
2044                                 gen_nullptr_check(s1);
2045                                 gen_bound_check;
2046                         }
2047                         x86_64_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
2048                         emit_store(jd, iptr, iptr->dst, d);
2049                         break;
2050
2051                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
2052
2053                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2054                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2055                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2056                         if (iptr->op1 == 0) {
2057                                 gen_nullptr_check(s1);
2058                                 gen_bound_check;
2059                         }
2060                         x86_64_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
2061                         emit_store(jd, iptr, iptr->dst, d);
2062                         break;
2063
2064                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
2065
2066                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2067                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2068                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
2069                         if (iptr->op1 == 0) {
2070                                 gen_nullptr_check(s1);
2071                                 gen_bound_check;
2072                         }
2073                         x86_64_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2074                         emit_store(jd, iptr, iptr->dst, d);
2075                         break;
2076
2077                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2078
2079                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2080                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2081                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP3);
2082                         if (iptr->op1 == 0) {
2083                                 gen_nullptr_check(s1);
2084                                 gen_bound_check;
2085                         }
2086                         x86_64_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2087                         emit_store(jd, iptr, iptr->dst, d);
2088                         break;
2089
2090                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
2091
2092                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2093                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2094                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2095                         if (iptr->op1 == 0) {
2096                                 gen_nullptr_check(s1);
2097                                 gen_bound_check;
2098                         }
2099                         x86_64_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
2100                         emit_store(jd, iptr, iptr->dst, d);
2101                         break;
2102
2103
2104                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2105
2106                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2107                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2108                         if (iptr->op1 == 0) {
2109                                 gen_nullptr_check(s1);
2110                                 gen_bound_check;
2111                         }
2112                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2113                         x86_64_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2114                         break;
2115
2116                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2117
2118                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2119                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2120                         if (iptr->op1 == 0) {
2121                                 gen_nullptr_check(s1);
2122                                 gen_bound_check;
2123                         }
2124                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2125                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2126                         break;
2127
2128                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2129
2130                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2131                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2132                         if (iptr->op1 == 0) {
2133                                 gen_nullptr_check(s1);
2134                                 gen_bound_check;
2135                         }
2136                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2137                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2138                         break;
2139
2140                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2141
2142                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2143                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2144                         if (iptr->op1 == 0) {
2145                                 gen_nullptr_check(s1);
2146                                 gen_bound_check;
2147                         }
2148                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2149                         x86_64_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2150                         break;
2151
2152                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2153
2154                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2155                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2156                         if (iptr->op1 == 0) {
2157                                 gen_nullptr_check(s1);
2158                                 gen_bound_check;
2159                         }
2160                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2161                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2162                         break;
2163
2164                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2165
2166                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2167                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2168                         if (iptr->op1 == 0) {
2169                                 gen_nullptr_check(s1);
2170                                 gen_bound_check;
2171                         }
2172                         s3 = emit_load_s3(jd, iptr, src, REG_FTMP3);
2173                         x86_64_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2174                         break;
2175
2176                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2177
2178                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2179                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2180                         if (iptr->op1 == 0) {
2181                                 gen_nullptr_check(s1);
2182                                 gen_bound_check;
2183                         }
2184                         s3 = emit_load_s3(jd, iptr, src, REG_FTMP3);
2185                         x86_64_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2186                         break;
2187
2188                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2189
2190                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2191                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2192                         if (iptr->op1 == 0) {
2193                                 gen_nullptr_check(s1);
2194                                 gen_bound_check;
2195                         }
2196                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2197
2198                         M_MOV(s1, rd->argintregs[0]);
2199                         M_MOV(s3, rd->argintregs[1]);
2200                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
2201                         M_CALL(REG_ITMP1);
2202                         M_TEST(REG_RESULT);
2203                         M_BEQ(0);
2204                         codegen_add_arraystoreexception_ref(cd);
2205
2206                         s1 = emit_load_s1(jd, iptr, src->prev->prev, REG_ITMP1);
2207                         s2 = emit_load_s2(jd, iptr, src->prev, REG_ITMP2);
2208                         s3 = emit_load_s3(jd, iptr, src, REG_ITMP3);
2209                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2210                         break;
2211
2212
2213                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2214
2215                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2216                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2217                         if (iptr->op1 == 0) {
2218                                 gen_nullptr_check(s1);
2219                                 gen_bound_check;
2220                         }
2221                         x86_64_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2222                         break;
2223
2224                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2225
2226                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2227                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2228                         if (iptr->op1 == 0) {
2229                                 gen_nullptr_check(s1);
2230                                 gen_bound_check;
2231                         }
2232                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2233                         break;
2234
2235                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2236
2237                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2238                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2239                         if (iptr->op1 == 0) {
2240                                 gen_nullptr_check(s1);
2241                                 gen_bound_check;
2242                         }
2243                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2244                         break;
2245
2246                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2247
2248                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2249                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2250                         if (iptr->op1 == 0) {
2251                                 gen_nullptr_check(s1);
2252                                 gen_bound_check;
2253                         }
2254                         x86_64_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2255                         break;
2256
2257                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2258
2259                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2260                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2261                         if (iptr->op1 == 0) {
2262                                 gen_nullptr_check(s1);
2263                                 gen_bound_check;
2264                         }
2265
2266                         if (IS_IMM32(iptr->val.l)) {
2267                                 x86_64_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2268                         } else {
2269                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2270                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2271                         }
2272                         break;
2273
2274                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2275
2276                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2277                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2278                         if (iptr->op1 == 0) {
2279                                 gen_nullptr_check(s1);
2280                                 gen_bound_check;
2281                         }
2282                         x86_64_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2283                         break;
2284
2285
2286                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2287                                       /* op1 = type, val.a = field address            */
2288
2289                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2290                                 disp = dseg_addaddress(cd, NULL);
2291
2292 /*                              PROFILE_CYCLE_STOP; */
2293
2294                                 codegen_addpatchref(cd, PATCHER_get_putstatic,
2295                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), disp);
2296
2297                                 if (opt_showdisassemble) {
2298                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2299                                 }
2300
2301 /*                              PROFILE_CYCLE_START; */
2302
2303                         } else {
2304                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr);
2305
2306                                 disp = dseg_addaddress(cd, &(fi->value));
2307
2308                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2309                                         PROFILE_CYCLE_STOP;
2310
2311                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2312
2313                                         if (opt_showdisassemble) {
2314                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2315                                         }
2316
2317                                         PROFILE_CYCLE_START;
2318                                 }
2319                         }
2320
2321                         /* This approach is much faster than moving the field
2322                            address inline into a register. */
2323
2324                         M_ALD(REG_ITMP2, RIP, -(((ptrint) cd->mcodeptr + 7) -
2325                                                                         (ptrint) cd->mcodebase) + disp);
2326
2327                         switch (iptr->op1) {
2328                         case TYPE_INT:
2329                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2330                                 M_ILD(d, REG_ITMP2, 0);
2331                                 emit_store(jd, iptr, iptr->dst, d);
2332                                 break;
2333                         case TYPE_LNG:
2334                         case TYPE_ADR:
2335                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2336                                 M_LLD(d, REG_ITMP2, 0);
2337                                 emit_store(jd, iptr, iptr->dst, d);
2338                                 break;
2339                         case TYPE_FLT:
2340                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2341                                 x86_64_movss_membase_reg(cd, REG_ITMP2, 0, d);
2342                                 emit_store(jd, iptr, iptr->dst, d);
2343                                 break;
2344                         case TYPE_DBL:
2345                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2346                                 x86_64_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2347                                 emit_store(jd, iptr, iptr->dst, d);
2348                                 break;
2349                         }
2350                         break;
2351
2352                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2353                                       /* op1 = type, val.a = field address            */
2354
2355                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2356                                 disp = dseg_addaddress(cd, NULL);
2357
2358 /*                              PROFILE_CYCLE_STOP; */
2359
2360                                 codegen_addpatchref(cd, PATCHER_get_putstatic,
2361                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), disp);
2362
2363                                 if (opt_showdisassemble) {
2364                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2365                                 }
2366
2367 /*                              PROFILE_CYCLE_START; */
2368
2369                         } else {
2370                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr);
2371
2372                                 disp = dseg_addaddress(cd, &(fi->value));
2373
2374                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2375                                         PROFILE_CYCLE_STOP;
2376
2377                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2378
2379                                         if (opt_showdisassemble) {
2380                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2381                                         }
2382
2383                                         PROFILE_CYCLE_START;
2384                                 }
2385                         }
2386
2387                         /* This approach is much faster than moving the field
2388                            address inline into a register. */
2389
2390                         M_ALD(REG_ITMP2, RIP, -(((ptrint) cd->mcodeptr + 7) -
2391                                                                         (ptrint) cd->mcodebase) + disp);
2392
2393                         switch (iptr->op1) {
2394                         case TYPE_INT:
2395                                 s2 = emit_load_s2(jd, iptr, src, REG_ITMP1);
2396                                 M_IST(s2, REG_ITMP2, 0);
2397                                 break;
2398                         case TYPE_LNG:
2399                         case TYPE_ADR:
2400                                 s2 = emit_load_s2(jd, iptr, src, REG_ITMP1);
2401                                 M_LST(s2, REG_ITMP2, 0);
2402                                 break;
2403                         case TYPE_FLT:
2404                                 s2 = emit_load_s2(jd, iptr, src, REG_FTMP1);
2405                                 x86_64_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2406                                 break;
2407                         case TYPE_DBL:
2408                                 s2 = emit_load_s2(jd, iptr, src, REG_FTMP1);
2409                                 x86_64_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2410                                 break;
2411                         }
2412                         break;
2413
2414                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2415                                           /* val = value (in current instruction)     */
2416                                           /* op1 = type, val.a = field address (in    */
2417                                           /* following NOP)                           */
2418
2419                         if (INSTRUCTION_IS_UNRESOLVED(iptr + 1)) {
2420                                 disp = dseg_addaddress(cd, NULL);
2421
2422 /*                              PROFILE_CYCLE_STOP; */
2423
2424                                 codegen_addpatchref(cd, PATCHER_get_putstatic,
2425                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr + 1), disp);
2426
2427                                 if (opt_showdisassemble) {
2428                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2429                                 }
2430
2431 /*                              PROFILE_CYCLE_START; */
2432
2433                         } else {
2434                                 fieldinfo *fi = INSTRUCTION_RESOLVED_FIELDINFO(iptr + 1);
2435
2436                                 disp = dseg_addaddress(cd, &(fi->value));
2437
2438                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2439                                         PROFILE_CYCLE_STOP;
2440
2441                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2442
2443                                         if (opt_showdisassemble) {
2444                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2445                                         }
2446
2447                                         PROFILE_CYCLE_START;
2448                                 }
2449                         }
2450
2451                         /* This approach is much faster than moving the field
2452                            address inline into a register. */
2453
2454                         M_ALD(REG_ITMP1, RIP, -(((ptrint) cd->mcodeptr + 7) -
2455                                                                         (ptrint) cd->mcodebase) + disp);
2456
2457                         switch (iptr->op1) {
2458                         case TYPE_INT:
2459                         case TYPE_FLT:
2460                                 M_IST_IMM(iptr->val.i, REG_ITMP1, 0);
2461                                 break;
2462                         case TYPE_LNG:
2463                         case TYPE_ADR:
2464                         case TYPE_DBL:
2465                                 if (IS_IMM32(iptr->val.l)) {
2466                                         M_LST_IMM32(iptr->val.l, REG_ITMP1, 0);
2467                                 } else {
2468                                         M_IST_IMM(iptr->val.l, REG_ITMP1, 0);
2469                                         M_IST_IMM(iptr->val.l >> 32, REG_ITMP1, 4);
2470                                 }
2471                                 break;
2472                         }
2473                         break;
2474
2475                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2476                                       /* op1 = type, val.i = field offset             */
2477
2478                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2479                         gen_nullptr_check(s1);
2480
2481                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2482 /*                              PROFILE_CYCLE_STOP; */
2483
2484                                 codegen_addpatchref(cd, PATCHER_get_putfield,
2485                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), 0);
2486
2487                                 if (opt_showdisassemble) {
2488                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2489                                 }
2490
2491 /*                              PROFILE_CYCLE_START; */
2492
2493                                 disp = 0;
2494
2495                         } else {
2496                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr)->offset;
2497                         }
2498
2499                         switch (iptr->op1) {
2500                         case TYPE_INT:
2501                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2502                                 M_ILD32(d, s1, disp);
2503                                 emit_store(jd, iptr, iptr->dst, d);
2504                                 break;
2505                         case TYPE_LNG:
2506                         case TYPE_ADR:
2507                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
2508                                 M_LLD32(d, s1, disp);
2509                                 emit_store(jd, iptr, iptr->dst, d);
2510                                 break;
2511                         case TYPE_FLT:
2512                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2513                                 x86_64_movss_membase32_reg(cd, s1, disp, d);
2514                                 emit_store(jd, iptr, iptr->dst, d);
2515                                 break;
2516                         case TYPE_DBL:
2517                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FTMP1);
2518                                 x86_64_movsd_membase32_reg(cd, s1, disp, d);
2519                                 emit_store(jd, iptr, iptr->dst, d);
2520                                 break;
2521                         }
2522                         break;
2523
2524                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2525                                       /* op1 = type, val.i = field offset             */
2526
2527                         s1 = emit_load_s1(jd, iptr, src->prev, REG_ITMP1);
2528                         gen_nullptr_check(s1);
2529
2530                         if (IS_INT_LNG_TYPE(iptr->op1)) {
2531                                 s2 = emit_load_s2(jd, iptr, src, REG_ITMP2);
2532                         } else {
2533                                 s2 = emit_load_s2(jd, iptr, src, REG_FTMP2);
2534                         }
2535
2536                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2537 /*                              PROFILE_CYCLE_STOP; */
2538
2539                                 codegen_addpatchref(cd, PATCHER_get_putfield,
2540                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr), 0);
2541
2542                                 if (opt_showdisassemble) {
2543                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2544                                 }
2545
2546 /*                              PROFILE_CYCLE_START; */
2547
2548                                 disp = 0;
2549
2550                         } else {
2551                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr)->offset;
2552                         }
2553
2554                         switch (iptr->op1) {
2555                         case TYPE_INT:
2556                                 M_IST32(s2, s1, disp);
2557                                 break;
2558                         case TYPE_LNG:
2559                         case TYPE_ADR:
2560                                 M_LST32(s2, s1, disp);
2561                                 break;
2562                         case TYPE_FLT:
2563                                 x86_64_movss_reg_membase32(cd, s2, s1, disp);
2564                                 break;
2565                         case TYPE_DBL:
2566                                 x86_64_movsd_reg_membase32(cd, s2, s1, disp);
2567                                 break;
2568                         }
2569                         break;
2570
2571                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2572                                           /* val = value (in current instruction)     */
2573                                           /* op1 = type, val.a = field address (in    */
2574                                           /* following NOP)                           */
2575
2576                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2577                         gen_nullptr_check(s1);
2578
2579                         if (INSTRUCTION_IS_UNRESOLVED(iptr + 1)) {
2580 /*                              PROFILE_CYCLE_STOP; */
2581
2582                                 codegen_addpatchref(cd, PATCHER_putfieldconst,
2583                                                                         INSTRUCTION_UNRESOLVED_FIELD(iptr + 1), 0);
2584
2585                                 if (opt_showdisassemble) {
2586                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2587                                 }
2588
2589 /*                              PROFILE_CYCLE_START; */
2590
2591                                 disp = 0;
2592
2593                         } else {
2594                                 disp = INSTRUCTION_RESOLVED_FIELDINFO(iptr + 1)->offset;
2595                         }
2596
2597                         switch (iptr->op1) {
2598                         case TYPE_INT:
2599                         case TYPE_FLT:
2600                                 M_IST32_IMM(iptr->val.i, s1, disp);
2601                                 break;
2602                         case TYPE_LNG:
2603                         case TYPE_ADR:
2604                         case TYPE_DBL:
2605                                 M_IST32_IMM(iptr->val.l, s1, disp);
2606                                 M_IST32_IMM(iptr->val.l >> 32, s1, disp + 4);
2607                                 break;
2608                         }
2609                         break;
2610
2611
2612                 /* branch operations **************************************************/
2613
2614                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2615
2616                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2617                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2618
2619                         PROFILE_CYCLE_STOP;
2620
2621 #ifdef ENABLE_VERIFIER
2622                         if (iptr->val.a) {
2623                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2624                                                                         (unresolved_class *) iptr->val.a, 0);
2625
2626                                 if (opt_showdisassemble) {
2627                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2628                                 }
2629                         }
2630 #endif /* ENABLE_VERIFIER */
2631
2632                         M_CALL_IMM(0);                            /* passing exception pc */
2633                         M_POP(REG_ITMP2_XPC);
2634
2635                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2636                         M_JMP(REG_ITMP3);
2637                         break;
2638
2639                 case ICMD_GOTO:         /* ... ==> ...                                */
2640                                         /* op1 = target JavaVM pc                     */
2641
2642                         M_JMP_IMM(0);
2643                         codegen_addreference(cd, (basicblock *) iptr->target);
2644                         break;
2645
2646                 case ICMD_JSR:          /* ... ==> ...                                */
2647                                         /* op1 = target JavaVM pc                     */
2648
2649                         M_CALL_IMM(0);
2650                         codegen_addreference(cd, (basicblock *) iptr->target);
2651                         break;
2652
2653                 case ICMD_RET:          /* ... ==> ...                                */
2654                                         /* op1 = local variable                       */
2655
2656                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2657                         if (var->flags & INMEMORY) {
2658                                 M_ALD(REG_ITMP1, REG_SP, var->regoff * 8);
2659                                 M_JMP(REG_ITMP1);
2660                         } else
2661                                 M_JMP(var->regoff);
2662                         break;
2663
2664                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2665                                         /* op1 = target JavaVM pc                     */
2666
2667                         if (src->flags & INMEMORY)
2668                                 M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
2669                         else
2670                                 M_TEST(src->regoff);
2671
2672                         /* If the conditional branch is part of an if-converted
2673                            block, don't generate the actual branch. */
2674
2675                         if ((iptr->opc & ICMD_CONDITION_MASK) == 0) {
2676                                 M_BEQ(0);
2677                                 codegen_addreference(cd, (basicblock *) iptr->target);
2678                         }
2679                         break;
2680
2681                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2682                                         /* op1 = target JavaVM pc                     */
2683
2684                         if (src->flags & INMEMORY)
2685                                 M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
2686                         else
2687                                 M_TEST(src->regoff);
2688
2689                         /* If the conditional branch is part of an if-converted
2690                            block, don't generate the actual branch. */
2691
2692                         if ((iptr->opc & ICMD_CONDITION_MASK) == 0) {
2693                                 M_BNE(0);
2694                                 codegen_addreference(cd, (basicblock *) iptr->target);
2695                         }
2696                         break;
2697
2698                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2699                                         /* op1 = target JavaVM pc, val.i = constant   */
2700
2701                         x86_64_emit_ifcc(cd, X86_64_CC_E, src, iptr);
2702                         break;
2703
2704                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2705                                         /* op1 = target JavaVM pc, val.i = constant   */
2706
2707                         x86_64_emit_ifcc(cd, X86_64_CC_L, src, iptr);
2708                         break;
2709
2710                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2711                                         /* op1 = target JavaVM pc, val.i = constant   */
2712
2713                         x86_64_emit_ifcc(cd, X86_64_CC_LE, src, iptr);
2714                         break;
2715
2716                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2717                                         /* op1 = target JavaVM pc, val.i = constant   */
2718
2719                         x86_64_emit_ifcc(cd, X86_64_CC_NE, src, iptr);
2720                         break;
2721
2722                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2723                                         /* op1 = target JavaVM pc, val.i = constant   */
2724
2725                         x86_64_emit_ifcc(cd, X86_64_CC_G, src, iptr);
2726                         break;
2727
2728                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2729                                         /* op1 = target JavaVM pc, val.i = constant   */
2730
2731                         x86_64_emit_ifcc(cd, X86_64_CC_GE, src, iptr);
2732                         break;
2733
2734                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2735                                         /* op1 = target JavaVM pc, val.l = constant   */
2736
2737                         x86_64_emit_if_lcc(cd, X86_64_CC_E, src, iptr);
2738                         break;
2739
2740                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2741                                         /* op1 = target JavaVM pc, val.l = constant   */
2742
2743                         x86_64_emit_if_lcc(cd, X86_64_CC_L, src, iptr);
2744                         break;
2745
2746                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2747                                         /* op1 = target JavaVM pc, val.l = constant   */
2748
2749                         x86_64_emit_if_lcc(cd, X86_64_CC_LE, src, iptr);
2750                         break;
2751
2752                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2753                                         /* op1 = target JavaVM pc, val.l = constant   */
2754
2755                         x86_64_emit_if_lcc(cd, X86_64_CC_NE, src, iptr);
2756                         break;
2757
2758                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2759                                         /* op1 = target JavaVM pc, val.l = constant   */
2760
2761                         x86_64_emit_if_lcc(cd, X86_64_CC_G, src, iptr);
2762                         break;
2763
2764                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2765                                         /* op1 = target JavaVM pc, val.l = constant   */
2766
2767                         x86_64_emit_if_lcc(cd, X86_64_CC_GE, src, iptr);
2768                         break;
2769
2770                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2771                                         /* op1 = target JavaVM pc                     */
2772
2773                         x86_64_emit_if_icmpcc(cd, X86_64_CC_E, src, iptr);
2774                         break;
2775
2776                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2777                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2778
2779                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_E, src, iptr);
2780                         break;
2781
2782                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2783                                         /* op1 = target JavaVM pc                     */
2784
2785                         x86_64_emit_if_icmpcc(cd, X86_64_CC_NE, src, iptr);
2786                         break;
2787
2788                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2789                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2790
2791                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_NE, src, iptr);
2792                         break;
2793
2794                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2795                                         /* op1 = target JavaVM pc                     */
2796
2797                         x86_64_emit_if_icmpcc(cd, X86_64_CC_L, src, iptr);
2798                         break;
2799
2800                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2801                                     /* op1 = target JavaVM pc                     */
2802
2803                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_L, src, iptr);
2804                         break;
2805
2806                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2807                                         /* op1 = target JavaVM pc                     */
2808
2809                         x86_64_emit_if_icmpcc(cd, X86_64_CC_G, src, iptr);
2810                         break;
2811
2812                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2813                                 /* op1 = target JavaVM pc                     */
2814
2815                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_G, src, iptr);
2816                         break;
2817
2818                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2819                                         /* op1 = target JavaVM pc                     */
2820
2821                         x86_64_emit_if_icmpcc(cd, X86_64_CC_LE, src, iptr);
2822                         break;
2823
2824                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2825                                         /* op1 = target JavaVM pc                     */
2826
2827                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_LE, src, iptr);
2828                         break;
2829
2830                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2831                                         /* op1 = target JavaVM pc                     */
2832
2833                         x86_64_emit_if_icmpcc(cd, X86_64_CC_GE, src, iptr);
2834                         break;
2835
2836                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2837                                     /* op1 = target JavaVM pc                     */
2838
2839                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_GE, src, iptr);
2840                         break;
2841
2842                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2843
2844                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2845                         break;
2846
2847                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2848                 case ICMD_IFNE_ICONST:  /* val.i = constant                           */
2849                 case ICMD_IFLT_ICONST:
2850                 case ICMD_IFGE_ICONST:
2851                 case ICMD_IFGT_ICONST:
2852                 case ICMD_IFLE_ICONST:
2853
2854                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
2855                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
2856                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2857                                 if (s1 == d) {
2858                                         M_INTMOVE(s1, REG_ITMP1);
2859                                         s1 = REG_ITMP1;
2860                                 }
2861                                 if (iptr[1].val.i == 0)
2862                                         M_CLR(d);
2863                                 else
2864                                         M_IMOV_IMM(iptr[1].val.i, d);
2865                         }
2866                         if (iptr->val.i == 0)
2867                                 M_CLR(REG_ITMP2);
2868                         else
2869                                 M_IMOV_IMM(iptr->val.i, REG_ITMP2);
2870                         M_ITEST(s1);
2871
2872                         switch (iptr->opc) {
2873                         case ICMD_IFEQ_ICONST:
2874                                 M_CMOVEQ(REG_ITMP2, d);
2875                                 break;
2876                         case ICMD_IFNE_ICONST:
2877                                 M_CMOVNE(REG_ITMP2, d);
2878                                 break;
2879                         case ICMD_IFLT_ICONST:
2880                                 M_CMOVLT(REG_ITMP2, d);
2881                                 break;
2882                         case ICMD_IFGE_ICONST:
2883                                 M_CMOVGE(REG_ITMP2, d);
2884                                 break;
2885                         case ICMD_IFGT_ICONST:
2886                                 M_CMOVGT(REG_ITMP2, d);
2887                                 break;
2888                         case ICMD_IFLE_ICONST:
2889                                 M_CMOVLE(REG_ITMP2, d);
2890                                 break;
2891                         }
2892
2893                         emit_store(jd, iptr, iptr->dst, d);
2894                         break;
2895
2896
2897                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2898                 case ICMD_LRETURN:
2899
2900                         s1 = emit_load_s1(jd, iptr, src, REG_RESULT);
2901                         M_INTMOVE(s1, REG_RESULT);
2902                         goto nowperformreturn;
2903
2904                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2905
2906                         s1 = emit_load_s1(jd, iptr, src, REG_RESULT);
2907                         M_INTMOVE(s1, REG_RESULT);
2908
2909 #ifdef ENABLE_VERIFIER
2910                         if (iptr->val.a) {
2911                                 PROFILE_CYCLE_STOP;
2912
2913                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2914                                                                         (unresolved_class *) iptr->val.a, 0);
2915
2916                                 if (opt_showdisassemble) {
2917                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2918                                 }
2919
2920                                 PROFILE_CYCLE_START;
2921                         }
2922 #endif /* ENABLE_VERIFIER */
2923                         goto nowperformreturn;
2924
2925                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2926                 case ICMD_DRETURN:
2927
2928                         s1 = emit_load_s1(jd, iptr, src, REG_FRESULT);
2929                         M_FLTMOVE(s1, REG_FRESULT);
2930                         goto nowperformreturn;
2931
2932                 case ICMD_RETURN:      /* ...  ==> ...                                */
2933
2934 nowperformreturn:
2935                         {
2936                         s4 i, p;
2937
2938                         p = stackframesize;
2939
2940 #if !defined(NDEBUG)
2941                         /* generate call trace */
2942
2943                         if (opt_verbosecall) {
2944                                 x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
2945
2946                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2947                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2948
2949                                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
2950                                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2951                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2952                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2953
2954                                 M_MOV_IMM(builtin_displaymethodstop, REG_ITMP1);
2955                                 M_CALL(REG_ITMP1);
2956
2957                                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2958                                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2959
2960                                 x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
2961                         }
2962 #endif /* !defined(NDEBUG) */
2963
2964 #if defined(USE_THREADS)
2965                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2966                                 M_ALD(rd->argintregs[0], REG_SP, rd->memuse * 8);
2967
2968                                 /* we need to save the proper return value */
2969                                 switch (iptr->opc) {
2970                                 case ICMD_IRETURN:
2971                                 case ICMD_ARETURN:
2972                                 case ICMD_LRETURN:
2973                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2974                                         break;
2975                                 case ICMD_FRETURN:
2976                                 case ICMD_DRETURN:
2977                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2978                                         break;
2979                                 }
2980
2981                                 M_MOV_IMM(builtin_monitorexit, REG_ITMP1);
2982                                 M_CALL(REG_ITMP1);
2983
2984                                 /* and now restore the proper return value */
2985                                 switch (iptr->opc) {
2986                                 case ICMD_IRETURN:
2987                                 case ICMD_ARETURN:
2988                                 case ICMD_LRETURN:
2989                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2990                                         break;
2991                                 case ICMD_FRETURN:
2992                                 case ICMD_DRETURN:
2993                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2994                                         break;
2995                                 }
2996                         }
2997 #endif
2998
2999                         /* restore saved registers */
3000
3001                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
3002                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
3003                         }
3004                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
3005                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
3006                         }
3007
3008                         /* deallocate stack */
3009
3010                         if (stackframesize)
3011                                 M_AADD_IMM(stackframesize * 8, REG_SP);
3012
3013                         /* generate method profiling code */
3014
3015                         PROFILE_CYCLE_STOP;
3016
3017                         M_RET;
3018                         }
3019                         break;
3020
3021
3022                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
3023                         {
3024                                 s4 i, l, *s4ptr;
3025                                 void **tptr;
3026
3027                                 tptr = (void **) iptr->target;
3028
3029                                 s4ptr = iptr->val.a;
3030                                 l = s4ptr[1];                          /* low     */
3031                                 i = s4ptr[2];                          /* high    */
3032
3033                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3034                                 M_INTMOVE(s1, REG_ITMP1);
3035                                 if (l != 0) {
3036                                         x86_64_alul_imm_reg(cd, X86_64_SUB, l, REG_ITMP1);
3037                                 }
3038                                 i = i - l + 1;
3039
3040                 /* range check */
3041                                 x86_64_alul_imm_reg(cd, X86_64_CMP, i - 1, REG_ITMP1);
3042                                 x86_64_jcc(cd, X86_64_CC_A, 0);
3043
3044                                 codegen_addreference(cd, (basicblock *) tptr[0]);
3045
3046                                 /* build jump table top down and use address of lowest entry */
3047
3048                 /* s4ptr += 3 + i; */
3049                                 tptr += i;
3050
3051                                 while (--i >= 0) {
3052                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
3053                                         --tptr;
3054                                 }
3055
3056                                 /* length of dataseg after last dseg_addtarget is used
3057                                    by load */
3058
3059                                 M_MOV_IMM(0, REG_ITMP2);
3060                                 dseg_adddata(cd);
3061                                 x86_64_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
3062                                 x86_64_jmp_reg(cd, REG_ITMP1);
3063                         }
3064                         break;
3065
3066
3067                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
3068                         {
3069                                 s4 i, l, val, *s4ptr;
3070                                 void **tptr;
3071
3072                                 tptr = (void **) iptr->target;
3073
3074                                 s4ptr = iptr->val.a;
3075                                 l = s4ptr[0];                          /* default  */
3076                                 i = s4ptr[1];                          /* count    */
3077
3078                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
3079                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);    /* reg compare should always be faster */
3080                                 while (--i >= 0) {
3081                                         s4ptr += 2;
3082                                         ++tptr;
3083
3084                                         val = s4ptr[0];
3085                                         x86_64_alul_imm_reg(cd, X86_64_CMP, val, s1);
3086                                         x86_64_jcc(cd, X86_64_CC_E, 0);
3087                                         codegen_addreference(cd, (basicblock *) tptr[0]);
3088                                 }
3089
3090                                 x86_64_jmp_imm(cd, 0);
3091
3092                                 tptr = (void **) iptr->target;
3093                                 codegen_addreference(cd, (basicblock *) tptr[0]);
3094                         }
3095                         break;
3096
3097
3098                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
3099                                         /* op1 = arg count val.a = builtintable entry */
3100
3101                         bte = iptr->val.a;
3102                         md = bte->md;
3103                         goto gen_method;
3104
3105                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
3106                                         /* op1 = arg count, val.a = method pointer    */
3107
3108                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
3109                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
3110                 case ICMD_INVOKEINTERFACE:
3111
3112                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3113                                 md = INSTRUCTION_UNRESOLVED_METHOD(iptr)->methodref->parseddesc.md;
3114                                 lm = NULL;
3115                         }
3116                         else {
3117                                 lm = INSTRUCTION_RESOLVED_METHODINFO(iptr);
3118                                 md = lm->parseddesc;
3119                         }
3120
3121 gen_method:
3122                         s3 = md->paramcount;
3123
3124                         MCODECHECK((20 * s3) + 128);
3125
3126                         /* copy arguments to registers or stack location */
3127
3128                         for (s3 = s3 - 1; s3 >= 0; s3--, src = src->prev) {
3129                                 if (src->varkind == ARGVAR)
3130                                         continue;
3131                                 if (IS_INT_LNG_TYPE(src->type)) {
3132                                         if (!md->params[s3].inmemory) {
3133                                                 s1 = rd->argintregs[md->params[s3].regoff];
3134                                                 d = emit_load_s1(jd, iptr, src, s1);
3135                                                 M_INTMOVE(d, s1);
3136                                         } else {
3137                                                 d = emit_load_s1(jd, iptr, src, REG_ITMP1);
3138                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
3139                                         }
3140
3141                                 } else {
3142                                         if (!md->params[s3].inmemory) {
3143                                                 s1 = rd->argfltregs[md->params[s3].regoff];
3144                                                 d = emit_load_s1(jd, iptr, src, s1);
3145                                                 M_FLTMOVE(d, s1);
3146                                         } else {
3147                                                 d = emit_load_s1(jd, iptr, src, REG_FTMP1);
3148                                                 M_DST(d, REG_SP, md->params[s3].regoff * 8);
3149                                         }
3150                                 }
3151                         }
3152
3153                         /* generate method profiling code */
3154
3155                         PROFILE_CYCLE_STOP;
3156
3157                         switch (iptr->opc) {
3158                         case ICMD_BUILTIN:
3159                                 a = (ptrint) bte->fp;
3160                                 d = md->returntype.type;
3161
3162                                 M_MOV_IMM(a, REG_ITMP1);
3163                                 M_CALL(REG_ITMP1);
3164
3165                                 /* if op1 == true, we need to check for an exception */
3166
3167                                 if (iptr->op1 == true) {
3168                                         M_TEST(REG_RESULT);
3169                                         M_BEQ(0);
3170                                         codegen_add_fillinstacktrace_ref(cd);
3171                                 }
3172                                 break;
3173
3174                         case ICMD_INVOKESPECIAL:
3175                                 M_TEST(rd->argintregs[0]);
3176                                 M_BEQ(0);
3177                                 codegen_add_nullpointerexception_ref(cd);
3178
3179                                 /* first argument contains pointer */
3180 /*                              gen_nullptr_check(rd->argintregs[0]); */
3181
3182                                 /* access memory for hardware nullptr */
3183 /*                              x86_64_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); */
3184
3185                                 /* fall through */
3186
3187                         case ICMD_INVOKESTATIC:
3188                                 if (lm == NULL) {
3189                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3190
3191                                         codegen_addpatchref(cd, PATCHER_invokestatic_special,
3192                                                                                 um, 0);
3193
3194                                         if (opt_showdisassemble) {
3195                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3196                                         }
3197
3198                                         a = 0;
3199                                         d = um->methodref->parseddesc.md->returntype.type;
3200
3201                                 } else {
3202                                         a = (ptrint) lm->stubroutine;
3203                                         d = lm->parseddesc->returntype.type;
3204                                 }
3205
3206                                 M_MOV_IMM(a, REG_ITMP2);
3207                                 M_CALL(REG_ITMP2);
3208                                 break;
3209
3210                         case ICMD_INVOKEVIRTUAL:
3211                                 gen_nullptr_check(rd->argintregs[0]);
3212
3213                                 if (lm == NULL) {
3214                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3215
3216                                         codegen_addpatchref(cd, PATCHER_invokevirtual, um, 0);
3217
3218                                         if (opt_showdisassemble) {
3219                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3220                                         }
3221
3222                                         s1 = 0;
3223                                         d = um->methodref->parseddesc.md->returntype.type;
3224
3225                                 } else {
3226                                         s1 = OFFSET(vftbl_t, table[0]) +
3227                                                 sizeof(methodptr) * lm->vftblindex;
3228                                         d = lm->parseddesc->returntype.type;
3229                                 }
3230
3231                                 M_ALD(REG_METHODPTR, rd->argintregs[0],
3232                                           OFFSET(java_objectheader, vftbl));
3233                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
3234                                 M_CALL(REG_ITMP3);
3235                                 break;
3236
3237                         case ICMD_INVOKEINTERFACE:
3238                                 gen_nullptr_check(rd->argintregs[0]);
3239
3240                                 if (lm == NULL) {
3241                                         unresolved_method *um = INSTRUCTION_UNRESOLVED_METHOD(iptr);
3242
3243                                         codegen_addpatchref(cd, PATCHER_invokeinterface, um, 0);
3244
3245                                         if (opt_showdisassemble) {
3246                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3247                                         }
3248
3249                                         s1 = 0;
3250                                         s2 = 0;
3251                                         d = um->methodref->parseddesc.md->returntype.type;
3252
3253                                 } else {
3254                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3255                                                 sizeof(methodptr) * lm->class->index;
3256
3257                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3258
3259                                         d = lm->parseddesc->returntype.type;
3260                                 }
3261
3262                                 M_ALD(REG_METHODPTR, rd->argintregs[0],
3263                                           OFFSET(java_objectheader, vftbl));
3264                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
3265                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
3266                                 M_CALL(REG_ITMP3);
3267                                 break;
3268                         }
3269
3270                         /* generate method profiling code */
3271
3272                         PROFILE_CYCLE_START;
3273
3274                         /* d contains return type */
3275
3276                         if (d != TYPE_VOID) {
3277                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3278                                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_RESULT);
3279                                         M_INTMOVE(REG_RESULT, s1);
3280                                         emit_store(jd, iptr, iptr->dst, s1);
3281                                 } else {
3282                                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_FRESULT);
3283                                         M_FLTMOVE(REG_FRESULT, s1);
3284                                         emit_store(jd, iptr, iptr->dst, s1);
3285                                 }
3286                         }
3287                         break;
3288
3289
3290                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3291
3292                                       /* op1:   0 == array, 1 == class                */
3293                                       /* val.a: (classinfo *) superclass              */
3294
3295                         /*  superclass is an interface:
3296                          *
3297                          *  OK if ((sub == NULL) ||
3298                          *         (sub->vftbl->interfacetablelength > super->index) &&
3299                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3300                          *
3301                          *  superclass is a class:
3302                          *
3303                          *  OK if ((sub == NULL) || (0
3304                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3305                          *         super->vftbl->diffval));
3306                          */
3307
3308                         if (iptr->op1 == 1) {
3309                                 /* object type cast-check */
3310
3311                                 classinfo *super;
3312                                 vftbl_t   *supervftbl;
3313                                 s4         superindex;
3314
3315                                 super = (classinfo *) iptr->val.a;
3316
3317                                 if (!super) {
3318                                         superindex = 0;
3319                                         supervftbl = NULL;
3320
3321                                 } else {
3322                                         superindex = super->index;
3323                                         supervftbl = super->vftbl;
3324                                 }
3325
3326 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3327                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3328 #endif
3329                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3330
3331                                 /* calculate interface checkcast code size */
3332
3333                                 s2 = 3; /* mov_membase_reg */
3334                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3335
3336                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3337                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3338                                         3 /* test */ + 6 /* jcc */;
3339
3340                                 if (!super)
3341                                         s2 += (opt_showdisassemble ? 5 : 0);
3342
3343                                 /* calculate class checkcast code size */
3344
3345                                 s3 = 3; /* mov_membase_reg */
3346                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3347                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3348
3349 #if 0
3350                                 if (s1 != REG_ITMP1) {
3351                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3352                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3353                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3354                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3355                                         a += 3;    /* sub */
3356
3357                                 } else
3358 #endif
3359                                         {
3360                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3361                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3362                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3363                                         }
3364
3365                                 s3 += 3 /* cmp */ + 6 /* jcc */;
3366
3367                                 if (!super)
3368                                         s3 += (opt_showdisassemble ? 5 : 0);
3369
3370                                 /* if class is not resolved, check which code to call */
3371
3372                                 if (!super) {
3373                                         M_TEST(s1);
3374                                         M_BEQ(6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3375
3376                                         codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3377                                                                                 (constant_classref *) iptr->target, 0);
3378
3379                                         if (opt_showdisassemble) {
3380                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3381                                         }
3382
3383                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
3384                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
3385                                         M_BEQ(s2 + 5);
3386                                 }
3387
3388                                 /* interface checkcast code */
3389
3390                                 if (!super || (super->flags & ACC_INTERFACE)) {
3391                                         if (super) {
3392                                                 M_TEST(s1);
3393                                                 M_BEQ(s2);
3394                                         }
3395
3396                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3397
3398                                         if (!super) {
3399                                                 codegen_addpatchref(cd,
3400                                                                                         PATCHER_checkcast_instanceof_interface,
3401                                                                                         (constant_classref *) iptr->target,
3402                                                                                         0);
3403
3404                                                 if (opt_showdisassemble) {
3405                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3406                                                 }
3407                                         }
3408
3409                                         x86_64_movl_membase32_reg(cd, REG_ITMP2,
3410                                                                                           OFFSET(vftbl_t, interfacetablelength),
3411                                                                                           REG_ITMP3);
3412                                         /* XXX TWISTI: should this be int arithmetic? */
3413                                         M_LSUB_IMM32(superindex, REG_ITMP3);
3414                                         M_TEST(REG_ITMP3);
3415                                         M_BLE(0);
3416                                         codegen_add_classcastexception_ref(cd);
3417                                         x86_64_mov_membase32_reg(cd, REG_ITMP2,
3418                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
3419                                                                                          superindex * sizeof(methodptr*),
3420                                                                                          REG_ITMP3);
3421                                         M_TEST(REG_ITMP3);
3422                                         M_BEQ(0);
3423                                         codegen_add_classcastexception_ref(cd);
3424
3425                                         if (!super)
3426                                                 M_JMP_IMM(s3);
3427                                 }
3428
3429                                 /* class checkcast code */
3430
3431                                 if (!super || !(super->flags & ACC_INTERFACE)) {
3432                                         if (super) {
3433                                                 M_TEST(s1);
3434                                                 M_BEQ(s3);
3435                                         }
3436
3437                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3438
3439                                         if (!super) {
3440                                                 codegen_addpatchref(cd, PATCHER_checkcast_class,
3441                                                                                         (constant_classref *) iptr->target,
3442                                                                                         0);
3443
3444                                                 if (opt_showdisassemble) {
3445                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3446                                                 }
3447                                         }
3448
3449                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3450 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3451                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3452 #endif
3453                                         x86_64_movl_membase32_reg(cd, REG_ITMP2,
3454                                                                                           OFFSET(vftbl_t, baseval),
3455                                                                                           REG_ITMP2);
3456                                         /*                                      if (s1 != REG_ITMP1) { */
3457                                         /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3458                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
3459                                         /*                                                                                              REG_ITMP1); */
3460                                         /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3461                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3462                                         /*                                                                                              REG_ITMP3); */
3463                                         /*  #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
3464                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3465                                         /*  #endif */
3466                                         /*                                              x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP1, REG_ITMP2); */
3467
3468                                         /*                                      } else { */
3469                                         x86_64_movl_membase32_reg(cd, REG_ITMP3,
3470                                                                                           OFFSET(vftbl_t, baseval),
3471                                                                                           REG_ITMP3);
3472                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3473                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3474                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3475                                         /*                                      } */
3476 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3477                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3478 #endif
3479                                         M_CMP(REG_ITMP3, REG_ITMP2);
3480                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3481                                         codegen_add_classcastexception_ref(cd);
3482                                 }
3483                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP3);
3484
3485                         } else {
3486                                 /* array type cast-check */
3487
3488                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3489                                 M_INTMOVE(s1, rd->argintregs[0]);
3490
3491                                 if (iptr->val.a == NULL) {
3492                                         codegen_addpatchref(cd, PATCHER_builtin_arraycheckcast,
3493                                                                                 (constant_classref *) iptr->target, 0);
3494
3495                                         if (opt_showdisassemble) {
3496                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3497                                         }
3498                                 }
3499
3500                                 M_MOV_IMM(iptr->val.a, rd->argintregs[1]);
3501                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
3502                                 M_CALL(REG_ITMP1);
3503                                 M_TEST(REG_RESULT);
3504                                 M_BEQ(0);
3505                                 codegen_add_classcastexception_ref(cd);
3506
3507                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3508                                 d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP1);
3509                         }
3510                         M_INTMOVE(s1, d);
3511                         emit_store(jd, iptr, iptr->dst, d);
3512                         break;
3513
3514                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3515
3516                                       /* op1:   0 == array, 1 == class                */
3517                                       /* val.a: (classinfo *) superclass              */
3518
3519                         /*  superclass is an interface:
3520                          *
3521                          *  return (sub != NULL) &&
3522                          *         (sub->vftbl->interfacetablelength > super->index) &&
3523                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3524                          *
3525                          *  superclass is a class:
3526                          *
3527                          *  return ((sub != NULL) && (0
3528                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3529                          *          super->vftbl->diffvall));
3530                          */
3531
3532                         {
3533                         classinfo *super;
3534                         vftbl_t   *supervftbl;
3535                         s4         superindex;
3536
3537                         super = (classinfo *) iptr->val.a;
3538
3539                         if (!super) {
3540                                 superindex = 0;
3541                                 supervftbl = NULL;
3542
3543                         } else {
3544                                 superindex = super->index;
3545                                 supervftbl = super->vftbl;
3546                         }
3547
3548 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3549             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3550 #endif
3551
3552                         s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3553                         d = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_ITMP2);
3554                         if (s1 == d) {
3555                                 M_INTMOVE(s1, REG_ITMP1);
3556                                 s1 = REG_ITMP1;
3557                         }
3558
3559                         /* calculate interface instanceof code size */
3560
3561                         s2 = 3; /* mov_membase_reg */
3562                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3563                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3564                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3565                                 3 /* test */ + 4 /* setcc */;
3566
3567                         if (!super)
3568                                 s2 += (opt_showdisassemble ? 5 : 0);
3569
3570                         /* calculate class instanceof code size */
3571
3572                         s3 = 3; /* mov_membase_reg */
3573                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3574                         s3 += 10; /* mov_imm_reg */
3575                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3576                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3577                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3578                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3579                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3580                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3581                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3582
3583                         if (!super)
3584                                 s3 += (opt_showdisassemble ? 5 : 0);
3585
3586                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3587
3588                         /* if class is not resolved, check which code to call */
3589
3590                         if (!super) {
3591                                 x86_64_test_reg_reg(cd, s1, s1);
3592                                 x86_64_jcc(cd, X86_64_CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3593                                                                                          7 + 6 + s2 + 5 + s3));
3594
3595                                 codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3596                                                                         (constant_classref *) iptr->target, 0);
3597
3598                                 if (opt_showdisassemble) {
3599                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3600                                 }
3601
3602                                 x86_64_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3603                                 x86_64_alul_imm_reg(cd, X86_64_AND, ACC_INTERFACE, REG_ITMP3);
3604                                 x86_64_jcc(cd, X86_64_CC_Z, s2 + 5);
3605                         }
3606
3607                         /* interface instanceof code */
3608
3609                         if (!super || (super->flags & ACC_INTERFACE)) {
3610                                 if (super) {
3611                                         x86_64_test_reg_reg(cd, s1, s1);
3612                                         x86_64_jcc(cd, X86_64_CC_Z, s2);
3613                                 }
3614
3615                                 x86_64_mov_membase_reg(cd, s1,
3616                                                                            OFFSET(java_objectheader, vftbl),
3617                                                                            REG_ITMP1);
3618                                 if (!super) {
3619                                         codegen_addpatchref(cd,
3620                                                                                 PATCHER_checkcast_instanceof_interface,
3621                                                                                 (constant_classref *) iptr->target, 0);
3622
3623                                         if (opt_showdisassemble) {
3624                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3625                                         }
3626                                 }
3627
3628                                 x86_64_movl_membase32_reg(cd, REG_ITMP1,
3629                                                                                   OFFSET(vftbl_t, interfacetablelength),
3630                                                                                   REG_ITMP3);
3631                                 x86_64_alu_imm32_reg(cd, X86_64_SUB, superindex, REG_ITMP3);
3632                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3633
3634                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3635
3636                                 x86_64_jcc(cd, X86_64_CC_LE, a);
3637                                 x86_64_mov_membase32_reg(cd, REG_ITMP1,
3638                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3639                                                                                  superindex * sizeof(methodptr*),
3640                                                                                  REG_ITMP1);
3641                                 x86_64_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3642                                 x86_64_setcc_reg(cd, X86_64_CC_NE, d);
3643
3644                                 if (!super)
3645                                         x86_64_jmp_imm(cd, s3);
3646                         }
3647
3648                         /* class instanceof code */
3649
3650                         if (!super || !(super->flags & ACC_INTERFACE)) {
3651                                 if (super) {
3652                                         x86_64_test_reg_reg(cd, s1, s1);
3653                                         x86_64_jcc(cd, X86_64_CC_E, s3);
3654                                 }
3655
3656                                 x86_64_mov_membase_reg(cd, s1,
3657                                                                            OFFSET(java_objectheader, vftbl),
3658                                                                            REG_ITMP1);
3659
3660                                 if (!super) {
3661                                         codegen_addpatchref(cd, PATCHER_instanceof_class,
3662                                                                                 (constant_classref *) iptr->target, 0);
3663
3664                                         if (opt_showdisassemble) {
3665                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3666                                         }
3667                                 }
3668
3669                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3670 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3671                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3672 #endif
3673                                 x86_64_movl_membase_reg(cd, REG_ITMP1,
3674                                                                                 OFFSET(vftbl_t, baseval),
3675                                                                                 REG_ITMP1);
3676                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3677                                                                                 OFFSET(vftbl_t, diffval),
3678                                                                                 REG_ITMP3);
3679                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3680                                                                                 OFFSET(vftbl_t, baseval),
3681                                                                                 REG_ITMP2);
3682 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3683                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3684 #endif
3685                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
3686                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d); /* may be REG_ITMP2 */
3687                                 x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP3, REG_ITMP1);
3688                                 x86_64_setcc_reg(cd, X86_64_CC_BE, d);
3689                         }
3690                         emit_store(jd, iptr, iptr->dst, d);
3691                         }
3692                         break;
3693
3694                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3695                                          /* op1 = dimension, val.a = class            */
3696
3697                         /* check for negative sizes and copy sizes to stack if necessary  */
3698
3699                         MCODECHECK((10 * 4 * iptr->op1) + 5 + 10 * 8);
3700
3701                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3702                                 /* copy SAVEDVAR sizes to stack */
3703
3704                                 if (src->varkind != ARGVAR) {
3705                                         s2 = emit_load_s2(jd, iptr, src, REG_ITMP1);
3706                                         M_LST(s2, REG_SP, s1 * 8);
3707                                 }
3708                         }
3709
3710                         /* is a patcher function set? */
3711
3712                         if (iptr->val.a == NULL) {
3713                                 codegen_addpatchref(cd, PATCHER_builtin_multianewarray,
3714                                                                         (constant_classref *) iptr->target, 0);
3715
3716                                 if (opt_showdisassemble) {
3717                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3718                                 }
3719
3720                                 a = 0;
3721
3722                         } else {
3723                                 a = (ptrint) iptr->val.a;
3724                         }
3725
3726                         /* a0 = dimension count */
3727
3728                         M_MOV_IMM(iptr->op1, rd->argintregs[0]);
3729
3730                         /* a1 = arrayvftbl */
3731
3732                         M_MOV_IMM(iptr->val.a, rd->argintregs[1]);
3733
3734                         /* a2 = pointer to dimensions = stack pointer */
3735
3736                         M_MOV(REG_SP, rd->argintregs[2]);
3737
3738                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3739                         M_CALL(REG_ITMP1);
3740
3741                         /* check for exception before result assignment */
3742
3743                         M_TEST(REG_RESULT);
3744                         M_BEQ(0);
3745                         codegen_add_fillinstacktrace_ref(cd);
3746
3747                         s1 = codegen_reg_of_var(rd, iptr->opc, iptr->dst, REG_RESULT);
3748                         M_INTMOVE(REG_RESULT, s1);
3749                         emit_store(jd, iptr, iptr->dst, s1);
3750                         break;
3751
3752                 default:
3753                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3754                         return false;
3755         } /* switch */
3756
3757         } /* for instruction */
3758
3759         /* copy values to interface registers */
3760
3761         src = bptr->outstack;
3762         len = bptr->outdepth;
3763         MCODECHECK(512);
3764 #if defined(ENABLE_LSRA)
3765         if (!opt_lsra)
3766 #endif
3767         while (src) {
3768                 len--;
3769                 if ((src->varkind != STACKVAR)) {
3770                         s2 = src->type;
3771                         if (IS_FLT_DBL_TYPE(s2)) {
3772                                 s1 = emit_load_s1(jd, iptr, src, REG_FTMP1);
3773                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3774                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3775
3776                                 } else {
3777                                         x86_64_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3778                                 }
3779
3780                         } else {
3781                                 s1 = emit_load_s1(jd, iptr, src, REG_ITMP1);
3782                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3783                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3784
3785                                 } else {
3786                                         x86_64_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3787                                 }
3788                         }
3789                 }
3790                 src = src->prev;
3791         }
3792
3793         /* At the end of a basic block we may have to append some nops,
3794            because the patcher stub calling code might be longer than the
3795            actual instruction. So codepatching does not change the
3796            following block unintentionally. */
3797
3798         if (cd->mcodeptr < cd->lastmcodeptr) {
3799                 while (cd->mcodeptr < cd->lastmcodeptr) {
3800                         M_NOP;
3801                 }
3802         }
3803
3804         } /* if (bptr -> flags >= BBREACHED) */
3805         } /* for basic block */
3806
3807         dseg_createlinenumbertable(cd);
3808
3809
3810         /* generate exception and patcher stubs */
3811
3812         {
3813                 exceptionref *eref;
3814                 patchref     *pref;
3815                 ptrint        mcode;
3816                 u1           *savedmcodeptr;
3817                 u1           *tmpmcodeptr;
3818
3819                 savedmcodeptr = NULL;
3820
3821                 /* generate exception stubs */
3822
3823                 for (eref = cd->exceptionrefs; eref != NULL; eref = eref->next) {
3824                         gen_resolvebranch(cd->mcodebase + eref->branchpos,
3825                                                           eref->branchpos,
3826                                                           cd->mcodeptr - cd->mcodebase);
3827
3828                         MCODECHECK(512);
3829
3830                         /* Check if the exception is an
3831                            ArrayIndexOutOfBoundsException.  If so, move index register
3832                            into a4. */
3833
3834                         if (eref->reg != -1)
3835                                 M_MOV(eref->reg, rd->argintregs[4]);
3836
3837                         /* calcuate exception address */
3838
3839                         M_MOV_IMM(0, rd->argintregs[3]);
3840                         dseg_adddata(cd);
3841                         M_AADD_IMM32(eref->branchpos - 6, rd->argintregs[3]);
3842
3843                         /* move function to call into REG_ITMP3 */
3844
3845                         M_MOV_IMM(eref->function, REG_ITMP3);
3846
3847                         if (savedmcodeptr != NULL) {
3848                                 M_JMP_IMM(savedmcodeptr - cd->mcodeptr - 5);
3849
3850                         } else {
3851                                 savedmcodeptr = cd->mcodeptr;
3852
3853                                 x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3854                                 M_MOV(REG_SP, rd->argintregs[1]);
3855                                 M_ALD(rd->argintregs[2], REG_SP, stackframesize * 8);
3856
3857                                 M_ASUB_IMM(2 * 8, REG_SP);
3858                                 M_AST(rd->argintregs[3], REG_SP, 0 * 8);         /* store XPC */
3859
3860                                 M_CALL(REG_ITMP3);
3861
3862                                 M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3863                                 M_AADD_IMM(2 * 8, REG_SP);
3864
3865                                 M_MOV_IMM(asm_handle_exception, REG_ITMP3);
3866                                 M_JMP(REG_ITMP3);
3867                         }
3868                 }
3869
3870
3871                 /* generate code patching stub call code */
3872
3873                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
3874                         /* check size of code segment */
3875
3876                         MCODECHECK(512);
3877
3878                         /* Get machine code which is patched back in later. A
3879                            `call rel32' is 5 bytes long (but read 8 bytes). */
3880
3881                         savedmcodeptr = cd->mcodebase + pref->branchpos;
3882                         mcode = *((ptrint *) savedmcodeptr);
3883
3884                         /* patch in `call rel32' to call the following code */
3885
3886                         tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr          */
3887                         cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position */
3888
3889                         M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
3890
3891                         cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr   */
3892
3893                         /* move pointer to java_objectheader onto stack */
3894
3895 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3896                         /* create a virtual java_objectheader */
3897
3898                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
3899                         a = dseg_addaddress(cd, NULL);                      /* vftbl      */
3900
3901                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP3);
3902                         M_PUSH(REG_ITMP3);
3903 #else
3904                         M_PUSH_IMM(0);
3905 #endif
3906
3907                         /* move machine code bytes and classinfo pointer into registers */
3908
3909                         M_MOV_IMM(mcode, REG_ITMP3);
3910                         M_PUSH(REG_ITMP3);
3911                         M_MOV_IMM(pref->ref, REG_ITMP3);
3912                         M_PUSH(REG_ITMP3);
3913                         M_MOV_IMM(pref->disp, REG_ITMP3);
3914                         M_PUSH(REG_ITMP3);
3915
3916                         M_MOV_IMM(pref->patcher, REG_ITMP3);
3917                         M_PUSH(REG_ITMP3);
3918
3919                         M_MOV_IMM(asm_wrapper_patcher, REG_ITMP3);
3920                         M_JMP(REG_ITMP3);
3921                 }
3922         }
3923
3924         /* generate replacement-out stubs */
3925
3926         {
3927                 int i;
3928
3929                 replacementpoint = jd->code->rplpoints;
3930
3931                 for (i = 0; i < jd->code->rplpointcount; ++i, ++replacementpoint) {
3932                         /* check code segment size */
3933
3934                         MCODECHECK(512);
3935
3936                         /* note start of stub code */
3937
3938                         replacementpoint->outcode = (u1*) (ptrint)(cd->mcodeptr - cd->mcodebase);
3939
3940                         /* make machine code for patching */
3941
3942                         disp = (ptrint)(replacementpoint->outcode - replacementpoint->pc) - 5;
3943                         replacementpoint->mcode = 0xe9 | ((u8)disp << 8);
3944
3945                         /* push address of `rplpoint` struct */
3946
3947                         M_MOV_IMM(replacementpoint, REG_ITMP3);
3948                         M_PUSH(REG_ITMP3);
3949
3950                         /* jump to replacement function */
3951
3952                         M_MOV_IMM(asm_replacement_out, REG_ITMP3);
3953                         M_JMP(REG_ITMP3);
3954                 }
3955         }
3956
3957         codegen_finish(jd);
3958
3959         /* everything's ok */
3960
3961         return true;
3962 }
3963
3964
3965 /* createcompilerstub **********************************************************
3966
3967    Creates a stub routine which calls the compiler.
3968
3969 *******************************************************************************/
3970
3971 #define COMPILERSTUB_DATASIZE    2 * SIZEOF_VOID_P
3972 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
3973
3974 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
3975
3976
3977 u1 *createcompilerstub(methodinfo *m)
3978 {
3979         u1          *s;                     /* memory to hold the stub            */
3980         ptrint      *d;
3981         codegendata *cd;
3982         s4           dumpsize;
3983
3984         s = CNEW(u1, COMPILERSTUB_SIZE);
3985
3986         /* set data pointer and code pointer */
3987
3988         d = (ptrint *) s;
3989         s = s + COMPILERSTUB_DATASIZE;
3990
3991         /* mark start of dump memory area */
3992
3993         dumpsize = dump_size();
3994
3995         cd = DNEW(codegendata);
3996         cd->mcodeptr = s;
3997
3998         /* Store the methodinfo* in the same place as in the methodheader
3999            for compiled methods. */
4000
4001         d[0] = (ptrint) asm_call_jit_compiler;
4002         d[1] = (ptrint) m;
4003
4004         /* code for the stub */
4005
4006         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 1 * SIZEOF_VOID_P));       /* methodinfo  */
4007         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 2 * SIZEOF_VOID_P));  /* compiler pointer */
4008         M_JMP(REG_ITMP3);
4009
4010 #if defined(ENABLE_STATISTICS)
4011         if (opt_stat)
4012                 count_cstub_len += COMPILERSTUB_SIZE;
4013 #endif
4014
4015         /* release dump area */
4016
4017         dump_release(dumpsize);
4018
4019         return s;
4020 }
4021
4022
4023 /* createnativestub ************************************************************
4024
4025    Creates a stub routine which calls a native method.
4026
4027 *******************************************************************************/
4028
4029 u1 *createnativestub(functionptr f, jitdata *jd, methoddesc *nmd)
4030 {
4031         methodinfo   *m;
4032         codegendata  *cd;
4033         registerdata *rd;
4034         methoddesc   *md;
4035         s4            stackframesize;       /* size of stackframe if needed       */
4036         s4            nativeparams;
4037         s4            i, j;                 /* count variables                    */
4038         s4            t;
4039         s4            s1, s2;
4040
4041         /* get required compiler data */
4042
4043         m  = jd->m;
4044         cd = jd->cd;
4045         rd = jd->rd;
4046
4047         /* initialize variables */
4048
4049         md = m->parseddesc;
4050         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
4051
4052         /* calculate stack frame size */
4053
4054         stackframesize =
4055                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
4056                 sizeof(localref_table) / SIZEOF_VOID_P +
4057                 INT_ARG_CNT + FLT_ARG_CNT + 1 +         /* + 1 for function address   */
4058                 nmd->memuse;
4059
4060         if (!(stackframesize & 0x1))                /* keep stack 16-byte aligned */
4061                 stackframesize++;
4062
4063         /* create method header */
4064
4065         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
4066         (void) dseg_adds4(cd, stackframesize * 8);              /* FrameSize      */
4067         (void) dseg_adds4(cd, 0);                               /* IsSync         */
4068         (void) dseg_adds4(cd, 0);                               /* IsLeaf         */
4069         (void) dseg_adds4(cd, 0);                               /* IntSave        */
4070         (void) dseg_adds4(cd, 0);                               /* FltSave        */
4071         (void) dseg_addlinenumbertablesize(cd);
4072         (void) dseg_adds4(cd, 0);                               /* ExTableSize    */
4073
4074         /* generate native method profiling code */
4075
4076         if (opt_prof) {
4077                 /* count frequency */
4078
4079                 M_MOV_IMM(m, REG_ITMP2);
4080                 M_IINC_MEMBASE(REG_ITMP2, OFFSET(methodinfo, frequency));
4081         }
4082
4083         /* generate stub code */
4084
4085         M_ASUB_IMM(stackframesize * 8, REG_SP);
4086
4087 #if !defined(NDEBUG)
4088         /* generate call trace */
4089
4090         if (opt_verbosecall) {
4091                 /* save integer and float argument registers */
4092
4093                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4094                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4095                                 M_LST(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4096
4097                 for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4098                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4099                                 M_DST(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4100
4101                 /* show integer hex code for float arguments */
4102
4103                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++) {
4104                         /* if the paramtype is a float, we have to right shift all
4105                            following integer registers */
4106
4107                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type)) {
4108                                 for (s1 = INT_ARG_CNT - 2; s1 >= i; s1--)
4109                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
4110
4111                                 x86_64_movd_freg_reg(cd, rd->argfltregs[j], rd->argintregs[i]);
4112                                 j++;
4113                         }
4114                 }
4115
4116                 M_MOV_IMM(m, REG_ITMP1);
4117                 M_AST(REG_ITMP1, REG_SP, 0 * 8);
4118                 M_MOV_IMM(builtin_trace_args, REG_ITMP1);
4119                 M_CALL(REG_ITMP1);
4120
4121                 /* restore integer and float argument registers */
4122
4123                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4124                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4125                                 M_LLD(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4126
4127                 for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4128                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4129                                 M_DLD(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4130         }
4131 #endif /* !defined(NDEBUG) */
4132
4133         /* get function address (this must happen before the stackframeinfo) */
4134
4135 #if !defined(WITH_STATIC_CLASSPATH)
4136         if (f == NULL) {
4137                 codegen_addpatchref(cd, PATCHER_resolve_native, m, 0);
4138
4139                 if (opt_showdisassemble) {
4140                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
4141                 }
4142         }
4143 #endif
4144
4145         M_MOV_IMM(f, REG_ITMP3);
4146
4147
4148         /* save integer and float argument registers */
4149
4150         for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4151                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4152                         M_LST(rd->argintregs[j++], REG_SP, i * 8);
4153
4154         for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4155                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4156                         M_DST(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4157
4158         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4159
4160         /* create dynamic stack info */
4161
4162         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4163         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[1]);
4164         M_ALEA(REG_SP, stackframesize * 8 + SIZEOF_VOID_P, rd->argintregs[2]);
4165         M_ALD(rd->argintregs[3], REG_SP, stackframesize * 8);
4166         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
4167         M_CALL(REG_ITMP1);
4168
4169         /* restore integer and float argument registers */
4170
4171         for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4172                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4173                         M_LLD(rd->argintregs[j++], REG_SP, i * 8);
4174
4175         for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4176                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4177                         M_DLD(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4178
4179         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4180
4181
4182         /* copy or spill arguments to new locations */
4183
4184         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
4185                 t = md->paramtypes[i].type;
4186
4187                 if (IS_INT_LNG_TYPE(t)) {
4188                         if (!md->params[i].inmemory) {
4189                                 s1 = rd->argintregs[md->params[i].regoff];
4190
4191                                 if (!nmd->params[j].inmemory) {
4192                                         s2 = rd->argintregs[nmd->params[j].regoff];
4193                                         M_INTMOVE(s1, s2);
4194
4195                                 } else {
4196                                         s2 = nmd->params[j].regoff;
4197                                         M_LST(s1, REG_SP, s2 * 8);
4198                                 }
4199
4200                         } else {
4201                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4202                                 s2 = nmd->params[j].regoff;
4203                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
4204                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
4205                         }
4206
4207                 } else {
4208                         /* We only copy spilled float arguments, as the float argument    */
4209                         /* registers keep unchanged.                                      */
4210
4211                         if (md->params[i].inmemory) {
4212                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4213                                 s2 = nmd->params[j].regoff;
4214                                 M_DLD(REG_FTMP1, REG_SP, s1 * 8);
4215                                 M_DST(REG_FTMP1, REG_SP, s2 * 8);
4216                         }
4217                 }
4218         }
4219
4220         /* put class into second argument register */
4221
4222         if (m->flags & ACC_STATIC)
4223                 M_MOV_IMM(m->class, rd->argintregs[1]);
4224
4225         /* put env into first argument register */
4226
4227         M_MOV_IMM(_Jv_env, rd->argintregs[0]);
4228
4229         /* do the native function call */
4230
4231         M_CALL(REG_ITMP3);
4232
4233         /* save return value */
4234
4235         if (md->returntype.type != TYPE_VOID) {
4236                 if (IS_INT_LNG_TYPE(md->returntype.type))
4237                         M_LST(REG_RESULT, REG_SP, 0 * 8);
4238                 else
4239                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
4240         }
4241
4242         /* remove native stackframe info */
4243
4244         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4245         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
4246         M_CALL(REG_ITMP1);
4247
4248 #if !defined(NDEBUG)
4249         /* generate call trace */
4250
4251         if (opt_verbosecall) {
4252                 /* just restore the value we need, don't care about the other */
4253
4254                 if (md->returntype.type != TYPE_VOID) {
4255                         if (IS_INT_LNG_TYPE(md->returntype.type))
4256                                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
4257                         else
4258                                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4259                 }
4260
4261                 M_MOV_IMM(m, rd->argintregs[0]);
4262                 M_MOV(REG_RESULT, rd->argintregs[1]);
4263                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4264                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4265
4266                 M_MOV_IMM(builtin_displaymethodstop, REG_ITMP1);
4267                 M_CALL(REG_ITMP1);
4268         }
4269 #endif /* !defined(NDEBUG) */
4270
4271         /* check for exception */
4272
4273 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4274         M_MOV_IMM(builtin_get_exceptionptrptr, REG_ITMP3);
4275         M_CALL(REG_ITMP3);
4276 #else
4277         M_MOV_IMM(&_no_threads_exceptionptr, REG_RESULT);
4278 #endif
4279         M_ALD(REG_ITMP2, REG_RESULT, 0);
4280
4281         /* restore return value */
4282
4283         if (md->returntype.type != TYPE_VOID) {
4284                 if (IS_INT_LNG_TYPE(md->returntype.type))
4285                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
4286                 else
4287                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4288         }
4289
4290         /* test for exception */
4291
4292         M_TEST(REG_ITMP2);
4293         M_BNE(7 + 1);
4294
4295         /* remove stackframe */
4296
4297         M_AADD_IMM(stackframesize * 8, REG_SP);
4298         M_RET;
4299
4300
4301         /* handle exception */
4302
4303 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4304         M_LST(REG_ITMP2, REG_SP, 0 * 8);
4305         M_MOV_IMM(builtin_get_exceptionptrptr, REG_ITMP3);
4306         M_CALL(REG_ITMP3);
4307         M_AST_IMM32(0, REG_RESULT, 0);                 /* clear exception pointer */
4308         M_LLD(REG_ITMP1_XPTR, REG_SP, 0 * 8);
4309 #else
4310         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
4311         M_MOV_IMM(&_no_threads_exceptionptr, REG_ITMP3);
4312         M_AST_IMM32(0, REG_ITMP3, 0);                  /* clear exception pointer */
4313 #endif
4314
4315         /* remove stackframe */
4316
4317         M_AADD_IMM(stackframesize * 8, REG_SP);
4318
4319         M_LLD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
4320         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
4321
4322         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
4323         M_JMP(REG_ITMP3);
4324
4325
4326         /* process patcher calls **************************************************/
4327
4328         {
4329                 patchref *pref;
4330                 ptrint    mcode;
4331                 u1       *savedmcodeptr;
4332                 u1       *tmpmcodeptr;
4333 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4334                 s4        disp;
4335 #endif
4336
4337                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4338                         /* Get machine code which is patched back in later. A
4339                            `call rel32' is 5 bytes long (but read 8 bytes). */
4340
4341                         savedmcodeptr = cd->mcodebase + pref->branchpos;
4342                         mcode = *((ptrint *) savedmcodeptr);
4343
4344                         /* patch in `call rel32' to call the following code               */
4345
4346                         tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr          */
4347                         cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position */
4348
4349                         M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
4350
4351                         cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr   */
4352
4353                         /* move pointer to java_objectheader onto stack */
4354
4355 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4356                         /* create a virtual java_objectheader */
4357
4358                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4359                         disp = dseg_addaddress(cd, NULL);                   /* vftbl      */
4360
4361                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP3);
4362                         M_PUSH(REG_ITMP3);
4363 #else
4364                         M_PUSH_IMM(0);
4365 #endif
4366
4367                         /* move machine code bytes and classinfo pointer into registers */
4368
4369                         M_MOV_IMM(mcode, REG_ITMP3);
4370                         M_PUSH(REG_ITMP3);
4371                         M_MOV_IMM(pref->ref, REG_ITMP3);
4372                         M_PUSH(REG_ITMP3);
4373                         M_MOV_IMM(pref->disp, REG_ITMP3);
4374                         M_PUSH(REG_ITMP3);
4375
4376                         M_MOV_IMM(pref->patcher, REG_ITMP3);
4377                         M_PUSH(REG_ITMP3);
4378
4379                         M_MOV_IMM(asm_wrapper_patcher, REG_ITMP3);
4380                         M_JMP(REG_ITMP3);
4381                 }
4382         }
4383
4384         codegen_finish(jd);
4385
4386         return jd->code->entrypoint;
4387 }
4388
4389
4390 /*
4391  * These are local overrides for various environment variables in Emacs.
4392  * Please do not remove this and leave it at the end of the file, where
4393  * Emacs will automagically detect them.
4394  * ---------------------------------------------------------------------
4395  * Local variables:
4396  * mode: c
4397  * indent-tabs-mode: t
4398  * c-basic-offset: 4
4399  * tab-width: 4
4400  * End:
4401  * vim:noexpandtab:sw=4:ts=4:
4402  */