src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31
  32    $Id: codegen.c 4508 2006-02-14 00:41:57Z twisti $
  33
  34 */
  35
  36
  37 #include "config.h"
  38
  39 #include <stdio.h>
  40
  41 #include "vm/types.h"
  42
  43 #include "md.h"
  44 #include "md-abi.h"
  45
  46 #include "vm/jit/x86_64/arch.h"
  47 #include "vm/jit/x86_64/codegen.h"
  48 #include "vm/jit/x86_64/emitfuncs.h"
  49
  50 #include "cacao/cacao.h"
  51 #include "native/native.h"
  52 #include "vm/builtin.h"
  53 #include "vm/exceptions.h"
  54 #include "vm/global.h"
  55 #include "vm/loader.h"
  56 #include "vm/options.h"
  57 #include "vm/statistics.h"
  58 #include "vm/stringlocal.h"
  59 #include "vm/jit/asmpart.h"
  60 #include "vm/jit/codegen-common.h"
  61 #include "vm/jit/dseg.h"
  62 #include "vm/jit/jit.h"
  63 #include "vm/jit/methodheader.h"
  64 #include "vm/jit/parse.h"
  65 #include "vm/jit/patcher.h"
  66 #include "vm/jit/reg.h"
  67
  68 #if defined(ENABLE_LSRA)
  69 # include "vm/jit/allocator/lsra.h"
  70 #endif
  71
  72
  73
  74
  75 /* codegen *********************************************************************
  76
  77    Generates machine code.
  78
  79 *******************************************************************************/
  80
  81 bool codegen(methodinfo *m, codegendata *cd, registerdata *rd)
  82 {
  83         s4                  len, s1, s2, s3, d, disp;
  84         u2                  currentline;
  85         ptrint              a;
  86         s4                  parentargs_base;
  87         stackptr            src;
  88         varinfo            *var;
  89         basicblock         *bptr;
  90         instruction        *iptr;
  91         exceptiontable     *ex;
  92         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
  93         builtintable_entry *bte;
  94         methoddesc         *md;
  95
  96         /* prevent compiler warnings */
  97
  98         d = 0;
  99         lm = NULL;
 100         bte = NULL;
 101
 102         {
 103         s4 i, p, t, l;
 104         s4 savedregs_num;
 105
 106         savedregs_num = 0;
 107
 108         /* space to save used callee saved registers */
 109
 110         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 111         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 112
 113         parentargs_base = rd->memuse + savedregs_num;
 114
 115 #if defined(USE_THREADS)
 116         /* space to save argument of monitor_enter */
 117
 118         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 119                 parentargs_base++;
 120 #endif
 121
 122     /* Keep stack of non-leaf functions 16-byte aligned for calls into native */
 123         /* code e.g. libc or jni (alignment problems with movaps).                */
 124
 125         if (!m->isleafmethod || runverbose)
 126                 parentargs_base |= 0x1;
 127
 128         /* create method header */
 129
 130         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 131         (void) dseg_adds4(cd, parentargs_base * 8);             /* FrameSize      */
 132
 133 #if defined(USE_THREADS)
 134         /* IsSync contains the offset relative to the stack pointer for the
 135            argument of monitor_exit used in the exception handler. Since the
 136            offset could be zero and give a wrong meaning of the flag it is
 137            offset by one.
 138         */
 139
 140         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 141                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);        /* IsSync         */
 142         else
 143 #endif
 144                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 145
 146         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 147         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse);  /* IntSave        */
 148         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse);  /* FltSave        */
 149
 150         (void) dseg_addlinenumbertablesize(cd);
 151
 152         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 153
 154         /* create exception table */
 155
 156         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 157                 dseg_addtarget(cd, ex->start);
 158                 dseg_addtarget(cd, ex->end);
 159                 dseg_addtarget(cd, ex->handler);
 160                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 161         }
 162
 163         /* initialize mcode variables */
 164
 165         cd->mcodeptr = (u1 *) cd->mcodebase;
 166         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
 167
 168         /* initialize the last patcher pointer */
 169
 170         cd->lastmcodeptr = cd->mcodeptr;
 171
 172         /* generate method profiling code */
 173
 174         if (opt_prof) {
 175                 /* count frequency */
 176
 177                 M_MOV_IMM((ptrint) m, REG_ITMP3);
 178                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(methodinfo, frequency));
 179
 180                 PROFILE_CYCLE_START;
 181         }
 182
 183         /* create stack frame (if necessary) */
 184
 185         if (parentargs_base)
 186                 M_ASUB_IMM(parentargs_base * 8, REG_SP);
 187
 188         /* save used callee saved registers */
 189
 190         p = parentargs_base;
 191         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 192                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 193         }
 194         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 195                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 196         }
 197
 198         /* take arguments out of register or stack frame */
 199
 200         md = m->parseddesc;
 201
 202         for (p = 0, l = 0; p < md->paramcount; p++) {
 203                 t = md->paramtypes[p].type;
 204                 var = &(rd->locals[l][t]);
 205                 l++;
 206                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 207                         l++;
 208                 if (var->type < 0)
 209                         continue;
 210                 s1 = md->params[p].regoff;
 211                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 212                         s2 = rd->argintregs[s1];
 213                         if (!md->params[p].inmemory) {           /* register arguments    */
 214                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 215                                         M_INTMOVE(s2, var->regoff);
 216
 217                                 } else {                             /* reg arg -> spilled    */
 218                                     M_LST(s2, REG_SP, var->regoff * 8);
 219                                 }
 220
 221                         } else {                                 /* stack arguments       */
 222                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 223                                         /* + 8 for return address */
 224                                         M_LLD(var->regoff, REG_SP, (parentargs_base + s1) * 8 + 8);
 225
 226                                 } else {                             /* stack arg -> spilled  */
 227                                         var->regoff = parentargs_base + s1 + 1;
 228                                 }
 229                         }
 230
 231                 } else {                                     /* floating args         */
 232                         if (!md->params[p].inmemory) {           /* register arguments    */
 233                                 s2 = rd->argfltregs[s1];
 234                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 235                                         M_FLTMOVE(s2, var->regoff);
 236
 237                                 } else {                                         /* reg arg -> spilled    */
 238                                         M_DST(s2, REG_SP, var->regoff * 8);
 239                                 }
 240
 241                         } else {                                 /* stack arguments       */
 242                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 243                                         M_DLD(var->regoff, REG_SP, (parentargs_base + s1) * 8 + 8);
 244
 245                                 } else {
 246                                         var->regoff = parentargs_base + s1 + 1;
 247                                 }
 248                         }
 249                 }
 250         }  /* end for */
 251
 252         /* save monitorenter argument */
 253
 254 #if defined(USE_THREADS)
 255         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 256                 /* stack offset for monitor argument */
 257
 258                 s1 = rd->memuse;
 259
 260                 if (runverbose) {
 261                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 262
 263                         for (p = 0; p < INT_ARG_CNT; p++)
 264                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 265
 266                         for (p = 0; p < FLT_ARG_CNT; p++)
 267                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 268
 269                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 270                 }
 271
 272                 /* decide which monitor enter function to call */
 273
 274                 if (m->flags & ACC_STATIC) {
 275                         M_MOV_IMM((ptrint) m->class, REG_ITMP1);
 276                         M_AST(REG_ITMP1, REG_SP, s1 * 8);
 277                         M_INTMOVE(REG_ITMP1, rd->argintregs[0]);
 278                         M_MOV_IMM((ptrint) BUILTIN_staticmonitorenter, REG_ITMP1);
 279                         M_CALL(REG_ITMP1);
 280
 281                 } else {
 282                         M_TEST(rd->argintregs[0]);
 283                         x86_64_jcc(cd, X86_64_CC_Z, 0);
 284                         codegen_addxnullrefs(cd, cd->mcodeptr);
 285                         M_AST(rd->argintregs[0], REG_SP, s1 * 8);
 286                         M_MOV_IMM((ptrint) BUILTIN_monitorenter, REG_ITMP1);
 287                         M_CALL(REG_ITMP1);
 288                 }
 289
 290                 if (runverbose) {
 291                         for (p = 0; p < INT_ARG_CNT; p++)
 292                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 293
 294                         for (p = 0; p < FLT_ARG_CNT; p++)
 295                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 296
 297                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 298                 }
 299         }
 300 #endif
 301
 302         /* Copy argument registers to stack and call trace function with
 303            pointer to arguments on stack. */
 304
 305         if (runverbose || opt_stat) {
 306                 M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 307
 308                 /* save integer argument registers */
 309
 310                 for (p = 0; p < INT_ARG_CNT; p++)
 311                         M_LST(rd->argintregs[p], REG_SP, (1 + p) * 8);
 312
 313                 /* save float argument registers */
 314
 315                 for (p = 0; p < FLT_ARG_CNT; p++)
 316                         M_DST(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 317
 318                 /* save temporary registers for leaf methods */
 319
 320                 if (m->isleafmethod) {
 321                         for (p = 0; p < INT_TMP_CNT; p++)
 322                                 M_LST(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 323
 324                         for (p = 0; p < FLT_TMP_CNT; p++)
 325                                 M_DST(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 326                 }
 327
 328                 if (runverbose) {
 329                         /* show integer hex code for float arguments */
 330
 331                         for (p = 0, l = 0; p < md->paramcount && p < INT_ARG_CNT; p++) {
 332                                 /* if the paramtype is a float, we have to right shift all    */
 333                                 /* following integer registers                                */
 334
 335                                 if (IS_FLT_DBL_TYPE(md->paramtypes[p].type)) {
 336                                         for (s1 = INT_ARG_CNT - 2; s1 >= p; s1--) {
 337                                                 M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
 338                                         }
 339
 340                                         x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 341                                         l++;
 342                                 }
 343                         }
 344
 345                         x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP2);
 346                         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_SP, 0 * 8);
 347                         x86_64_mov_imm_reg(cd, (ptrint) builtin_trace_args, REG_ITMP1);
 348                         x86_64_call_reg(cd, REG_ITMP1);
 349                 }
 350
 351                 /* restore integer argument registers */
 352
 353                 for (p = 0; p < INT_ARG_CNT; p++)
 354                         M_LLD(rd->argintregs[p], REG_SP, (1 + p) * 8);
 355
 356                 /* restore float argument registers */
 357
 358                 for (p = 0; p < FLT_ARG_CNT; p++)
 359                         M_DLD(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 360
 361                 /* restore temporary registers for leaf methods */
 362
 363                 if (m->isleafmethod) {
 364                         for (p = 0; p < INT_TMP_CNT; p++)
 365                                 M_LLD(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 366
 367                         for (p = 0; p < FLT_TMP_CNT; p++)
 368                                 M_DLD(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 369                 }
 370
 371                 M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 372         }
 373
 374         }
 375
 376         /* end of header generation */
 377
 378         /* walk through all basic blocks */
 379
 380         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 381
 382                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 383
 384                 if (bptr->flags >= BBREACHED) {
 385
 386                         /* branch resolving */
 387
 388                         branchref *bref;
 389                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 390                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 391                                                                   bref->branchpos,
 392                                                                   bptr->mpc);
 393                         }
 394
 395                 /* copy interface registers to their destination */
 396
 397                 src = bptr->instack;
 398                 len = bptr->indepth;
 399                 MCODECHECK(512);
 400
 401                 /* generate basicblock profiling code */
 402
 403                 if (opt_prof_bb) {
 404                         /* count frequency */
 405
 406                         M_MOV_IMM((ptrint) m->bbfrequency, REG_ITMP2);
 407                         M_IINC_MEMBASE(REG_ITMP2, bptr->debug_nr * 4);
 408
 409                         /* if this is an exception handler, start profiling again */
 410
 411                         if (bptr->type == BBTYPE_EXH)
 412                                 PROFILE_CYCLE_START;
 413                 }
 414
 415 #if defined(ENABLE_LSRA)
 416                 if (opt_lsra) {
 417                         while (src != NULL) {
 418                                 len--;
 419                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 420                                         if (bptr->type == BBTYPE_SBR) {
 421                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 422                                                 if (!(src->flags & INMEMORY))
 423                                                         d= src->regoff;
 424                                                 else
 425                                                         d=REG_ITMP1;
 426                                                 x86_64_pop_reg(cd, d);
 427                                                 store_reg_to_var_int(src, d);
 428
 429                                         } else if (bptr->type == BBTYPE_EXH) {
 430                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 431                                                 if (!(src->flags & INMEMORY))
 432                                                         d= src->regoff;
 433                                                 else
 434                                                         d=REG_ITMP1;
 435                                                 M_INTMOVE(REG_ITMP1, d);
 436                                                 store_reg_to_var_int(src, d);
 437                                         }
 438                                 }
 439                                 src = src->prev;
 440                         }
 441
 442                 } else {
 443 #endif
 444
 445                 while (src != NULL) {
 446                         len--;
 447                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 448                                 if (bptr->type == BBTYPE_SBR) {
 449                                         d = reg_of_var(rd, src, REG_ITMP1);
 450                                         M_POP(d);
 451                                         store_reg_to_var_int(src, d);
 452
 453                                 } else if (bptr->type == BBTYPE_EXH) {
 454                                         d = reg_of_var(rd, src, REG_ITMP1);
 455                                         M_INTMOVE(REG_ITMP1, d);
 456                                         store_reg_to_var_int(src, d);
 457                                 }
 458
 459                         } else {
 460                                 d = reg_of_var(rd, src, REG_ITMP1);
 461                                 if ((src->varkind != STACKVAR)) {
 462                                         s2 = src->type;
 463                                         if (IS_FLT_DBL_TYPE(s2)) {
 464                                                 s1 = rd->interfaces[len][s2].regoff;
 465
 466                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 467                                                         M_FLTMOVE(s1, d);
 468                                                 else
 469                                                         M_DLD(d, REG_SP, s1 * 8);
 470
 471                                                 store_reg_to_var_flt(src, d);
 472
 473                                         } else {
 474                                                 s1 = rd->interfaces[len][s2].regoff;
 475
 476                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 477                                                         M_INTMOVE(s1, d);
 478                                                 else
 479                                                         M_LLD(d, REG_SP, s1 * 8);
 480
 481                                                 store_reg_to_var_int(src, d);
 482                                         }
 483                                 }
 484                         }
 485                         src = src->prev;
 486                 }
 487 #if defined(ENABLE_LSRA)
 488                 }
 489 #endif
 490                 /* walk through all instructions */
 491
 492                 src = bptr->instack;
 493                 len = bptr->icount;
 494                 currentline = 0;
 495
 496                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 497                         if (iptr->line != currentline) {
 498                                 dseg_addlinenumber(cd, iptr->line, cd->mcodeptr);
 499                                 currentline = iptr->line;
 500                         }
 501
 502                         MCODECHECK(1024);                         /* 1KB should be enough */
 503
 504                         switch (iptr->opc) {
 505                         case ICMD_INLINE_START: /* internal ICMDs                         */
 506                         case ICMD_INLINE_END:
 507                                 break;
 508
 509                         case ICMD_NOP:    /* ...  ==> ...                                 */
 510                                 break;
 511
 512                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 513
 514                                 if (src->flags & INMEMORY)
 515                                         M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
 516                                 else
 517                                         M_TEST(src->regoff);
 518                                 M_BEQ(0);
 519                                 codegen_addxnullrefs(cd, cd->mcodeptr);
 520                                 break;
 521
 522                 /* constant operations ************************************************/
 523
 524                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 525                                       /* op1 = 0, val.i = constant                    */
 526
 527                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 528                         if (iptr->val.i == 0)
 529                                 M_CLR(d);
 530                         else
 531                                 M_IMOV_IMM(iptr->val.i, d);
 532                         store_reg_to_var_int(iptr->dst, d);
 533                         break;
 534
 535                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 536                                       /* op1 = 0, val.l = constant                    */
 537
 538                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 539                         if (iptr->val.l == 0)
 540                                 M_CLR(d);
 541                         else
 542                                 M_MOV_IMM(iptr->val.l, d);
 543                         store_reg_to_var_int(iptr->dst, d);
 544                         break;
 545
 546                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 547                                       /* op1 = 0, val.f = constant                    */
 548
 549                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 550                         disp = dseg_addfloat(cd, iptr->val.f);
 551                         x86_64_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + disp, d);
 552                         store_reg_to_var_flt(iptr->dst, d);
 553                         break;
 554
 555                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 556                                       /* op1 = 0, val.d = constant                    */
 557
 558                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 559                         disp = dseg_adddouble(cd, iptr->val.d);
 560                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, d);
 561                         store_reg_to_var_flt(iptr->dst, d);
 562                         break;
 563
 564                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 565                                       /* op1 = 0, val.a = constant                    */
 566
 567                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 568
 569                         if ((iptr->target != NULL) && (iptr->val.a == NULL)) {
 570 /*                              PROFILE_CYCLE_STOP; */
 571
 572                                 codegen_addpatchref(cd, cd->mcodeptr,
 573                                                                         PATCHER_aconst,
 574                                                                         (unresolved_class *) iptr->target, 0);
 575
 576                                 if (opt_showdisassemble) {
 577                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
 578                                 }
 579
 580 /*                              PROFILE_CYCLE_START; */
 581
 582                                 M_MOV_IMM((ptrint) iptr->val.a, d);
 583
 584                         } else {
 585                                 if (iptr->val.a == 0)
 586                                         M_CLR(d);
 587                                 else
 588                                         M_MOV_IMM((ptrint) iptr->val.a, d);
 589                         }
 590                         store_reg_to_var_int(iptr->dst, d);
 591                         break;
 592
 593
 594                 /* load/store operations **********************************************/
 595
 596                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 597                                       /* op1 = local variable                         */
 598
 599                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 600                         if ((iptr->dst->varkind == LOCALVAR) &&
 601                             (iptr->dst->varnum == iptr->op1)) {
 602                                 break;
 603                         }
 604                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 605                         if (var->flags & INMEMORY) {
 606                                 x86_64_movl_membase_reg(cd, REG_SP, var->regoff * 8, d);
 607                                 store_reg_to_var_int(iptr->dst, d);
 608
 609                         } else {
 610                                 if (iptr->dst->flags & INMEMORY) {
 611                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 612
 613                                 } else {
 614                                         M_INTMOVE(var->regoff, d);
 615                                 }
 616                         }
 617                         break;
 618
 619                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 620                 case ICMD_ALOAD:      /* op1 = local variable                         */
 621
 622                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 623                         if ((iptr->dst->varkind == LOCALVAR) &&
 624                             (iptr->dst->varnum == iptr->op1)) {
 625                                 break;
 626                         }
 627                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 628                         if (var->flags & INMEMORY) {
 629                                 x86_64_mov_membase_reg(cd, REG_SP, var->regoff * 8, d);
 630                                 store_reg_to_var_int(iptr->dst, d);
 631
 632                         } else {
 633                                 if (iptr->dst->flags & INMEMORY) {
 634                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 635
 636                                 } else {
 637                                         M_INTMOVE(var->regoff, d);
 638                                 }
 639                         }
 640                         break;
 641
 642                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 643                 case ICMD_DLOAD:      /* op1 = local variable                         */
 644
 645                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 646                         if ((iptr->dst->varkind == LOCALVAR) &&
 647                             (iptr->dst->varnum == iptr->op1)) {
 648                                 break;
 649                         }
 650                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 651                         if (var->flags & INMEMORY) {
 652                                 x86_64_movq_membase_reg(cd, REG_SP, var->regoff * 8, d);
 653                                 store_reg_to_var_flt(iptr->dst, d);
 654
 655                         } else {
 656                                 if (iptr->dst->flags & INMEMORY) {
 657                                         x86_64_movq_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 658
 659                                 } else {
 660                                         M_FLTMOVE(var->regoff, d);
 661                                 }
 662                         }
 663                         break;
 664
 665                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 666                 case ICMD_LSTORE:     /* op1 = local variable                         */
 667                 case ICMD_ASTORE:
 668
 669                         if ((src->varkind == LOCALVAR) &&
 670                             (src->varnum == iptr->op1)) {
 671                                 break;
 672                         }
 673                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 674                         if (var->flags & INMEMORY) {
 675                                 var_to_reg_int(s1, src, REG_ITMP1);
 676                                 x86_64_mov_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 677
 678                         } else {
 679                                 var_to_reg_int(s1, src, var->regoff);
 680                                 M_INTMOVE(s1, var->regoff);
 681                         }
 682                         break;
 683
 684                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 685                 case ICMD_DSTORE:     /* op1 = local variable                         */
 686
 687                         if ((src->varkind == LOCALVAR) &&
 688                             (src->varnum == iptr->op1)) {
 689                                 break;
 690                         }
 691                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 692                         if (var->flags & INMEMORY) {
 693                                 var_to_reg_flt(s1, src, REG_FTMP1);
 694                                 x86_64_movq_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 695
 696                         } else {
 697                                 var_to_reg_flt(s1, src, var->regoff);
 698                                 M_FLTMOVE(s1, var->regoff);
 699                         }
 700                         break;
 701
 702
 703                 /* pop/dup/swap operations ********************************************/
 704
 705                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 706
 707                 case ICMD_POP:        /* ..., value  ==> ...                          */
 708                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 709                         break;
 710
 711                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 712                         M_COPY(src, iptr->dst);
 713                         break;
 714
 715                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 716
 717                         M_COPY(src,       iptr->dst);
 718                         M_COPY(src->prev, iptr->dst->prev);
 719                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 720                         break;
 721
 722                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 723
 724                         M_COPY(src,             iptr->dst);
 725                         M_COPY(src->prev,       iptr->dst->prev);
 726                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 727                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 728                         break;
 729
 730                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 731
 732                         M_COPY(src,       iptr->dst);
 733                         M_COPY(src->prev, iptr->dst->prev);
 734                         break;
 735
 736                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 737
 738                         M_COPY(src,             iptr->dst);
 739                         M_COPY(src->prev,       iptr->dst->prev);
 740                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 741                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 742                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 743                         break;
 744
 745                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 746
 747                         M_COPY(src,                   iptr->dst);
 748                         M_COPY(src->prev,             iptr->dst->prev);
 749                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 750                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 751                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 752                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 753                         break;
 754
 755                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 756
 757                         M_COPY(src,       iptr->dst->prev);
 758                         M_COPY(src->prev, iptr->dst);
 759                         break;
 760
 761
 762                 /* integer operations *************************************************/
 763
 764                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 765
 766                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 767                         if (iptr->dst->flags & INMEMORY) {
 768                                 if (src->flags & INMEMORY) {
 769                                         if (src->regoff == iptr->dst->regoff) {
 770                                                 x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 771
 772                                         } else {
 773                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 774                                                 x86_64_negl_reg(cd, REG_ITMP1);
 775                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 776                                         }
 777
 778                                 } else {
 779                                         x86_64_movl_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 780                                         x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 781                                 }
 782
 783                         } else {
 784                                 if (src->flags & INMEMORY) {
 785                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 786                                         x86_64_negl_reg(cd, d);
 787
 788                                 } else {
 789                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 790                                         x86_64_negl_reg(cd, iptr->dst->regoff);
 791                                 }
 792                         }
 793                         break;
 794
 795                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 796
 797                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 798                         if (iptr->dst->flags & INMEMORY) {
 799                                 if (src->flags & INMEMORY) {
 800                                         if (src->regoff == iptr->dst->regoff) {
 801                                                 x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 802
 803                                         } else {
 804                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 805                                                 x86_64_neg_reg(cd, REG_ITMP1);
 806                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 807                                         }
 808
 809                                 } else {
 810                                         x86_64_mov_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 811                                         x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 812                                 }
 813
 814                         } else {
 815                                 if (src->flags & INMEMORY) {
 816                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 817                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 818
 819                                 } else {
 820                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 821                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 822                                 }
 823                         }
 824                         break;
 825
 826                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 827
 828                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 829                         if (src->flags & INMEMORY) {
 830                                 x86_64_movslq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 831
 832                         } else {
 833                                 x86_64_movslq_reg_reg(cd, src->regoff, d);
 834                         }
 835                         store_reg_to_var_int(iptr->dst, d);
 836                         break;
 837
 838                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 839
 840                         var_to_reg_int(s1, src, REG_ITMP1);
 841                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 842                         M_INTMOVE(s1, d);
 843                         store_reg_to_var_int(iptr->dst, d);
 844                         break;
 845
 846                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 847
 848                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 849                         if (src->flags & INMEMORY) {
 850                                 x86_64_movsbq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 851
 852                         } else {
 853                                 x86_64_movsbq_reg_reg(cd, src->regoff, d);
 854                         }
 855                         store_reg_to_var_int(iptr->dst, d);
 856                         break;
 857
 858                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 859
 860                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 861                         if (src->flags & INMEMORY) {
 862                                 x86_64_movzwq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 863
 864                         } else {
 865                                 x86_64_movzwq_reg_reg(cd, src->regoff, d);
 866                         }
 867                         store_reg_to_var_int(iptr->dst, d);
 868                         break;
 869
 870                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 871
 872                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 873                         if (src->flags & INMEMORY) {
 874                                 x86_64_movswq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 875
 876                         } else {
 877                                 x86_64_movswq_reg_reg(cd, src->regoff, d);
 878                         }
 879                         store_reg_to_var_int(iptr->dst, d);
 880                         break;
 881
 882
 883                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 884
 885                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 886                         x86_64_emit_ialu(cd, X86_64_ADD, src, iptr);
 887                         break;
 888
 889                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 890                                       /* val.i = constant                             */
 891
 892                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 893                         x86_64_emit_ialuconst(cd, X86_64_ADD, src, iptr);
 894                         break;
 895
 896                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 897
 898                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 899                         x86_64_emit_lalu(cd, X86_64_ADD, src, iptr);
 900                         break;
 901
 902                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 903                                       /* val.l = constant                             */
 904
 905                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 906                         x86_64_emit_laluconst(cd, X86_64_ADD, src, iptr);
 907                         break;
 908
 909                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 910
 911                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 912                         if (iptr->dst->flags & INMEMORY) {
 913                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 914                                         if (src->prev->regoff == iptr->dst->regoff) {
 915                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 916                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 917
 918                                         } else {
 919                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 920                                                 x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 921                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 922                                         }
 923
 924                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 925                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 926                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 927                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 928
 929                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 930                                         if (src->prev->regoff == iptr->dst->regoff) {
 931                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 932
 933                                         } else {
 934                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 935                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 936                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 937                                         }
 938
 939                                 } else {
 940                                         x86_64_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 941                                         x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 942                                 }
 943
 944                         } else {
 945                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 946                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 947                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 948
 949                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 950                                         M_INTMOVE(src->prev->regoff, d);
 951                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 952
 953                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 954                                         /* workaround for reg alloc */
 955                                         if (src->regoff == iptr->dst->regoff) {
 956                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 957                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 958                                                 M_INTMOVE(REG_ITMP1, d);
 959
 960                                         } else {
 961                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 962                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 963                                         }
 964
 965                                 } else {
 966                                         /* workaround for reg alloc */
 967                                         if (src->regoff == iptr->dst->regoff) {
 968                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
 969                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 970                                                 M_INTMOVE(REG_ITMP1, d);
 971
 972                                         } else {
 973                                                 M_INTMOVE(src->prev->regoff, d);
 974                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 975                                         }
 976                                 }
 977                         }
 978                         break;
 979
 980                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 981                                       /* val.i = constant                             */
 982
 983                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 984                         x86_64_emit_ialuconst(cd, X86_64_SUB, src, iptr);
 985                         break;
 986
 987                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 988
 989                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 990                         if (iptr->dst->flags & INMEMORY) {
 991                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 992                                         if (src->prev->regoff == iptr->dst->regoff) {
 993                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 994                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 995
 996                                         } else {
 997                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 998                                                 x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 999                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1000                                         }
1001
1002                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1003                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1004                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1005                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1006
1007                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1008                                         if (src->prev->regoff == iptr->dst->regoff) {
1009                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1010
1011                                         } else {
1012                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1013                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1014                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1015                                         }
1016
1017                                 } else {
1018                                         x86_64_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
1019                                         x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1020                                 }
1021
1022                         } else {
1023                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1024                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1025                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1026
1027                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1028                                         M_INTMOVE(src->prev->regoff, d);
1029                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1030
1031                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1032                                         /* workaround for reg alloc */
1033                                         if (src->regoff == iptr->dst->regoff) {
1034                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1035                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1036                                                 M_INTMOVE(REG_ITMP1, d);
1037
1038                                         } else {
1039                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1040                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1041                                         }
1042
1043                                 } else {
1044                                         /* workaround for reg alloc */
1045                                         if (src->regoff == iptr->dst->regoff) {
1046                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1047                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1048                                                 M_INTMOVE(REG_ITMP1, d);
1049
1050                                         } else {
1051                                                 M_INTMOVE(src->prev->regoff, d);
1052                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1053                                         }
1054                                 }
1055                         }
1056                         break;
1057
1058                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1059                                       /* val.l = constant                             */
1060
1061                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1062                         x86_64_emit_laluconst(cd, X86_64_SUB, src, iptr);
1063                         break;
1064
1065                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1066
1067                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1068                         if (iptr->dst->flags & INMEMORY) {
1069                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1070                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1071                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1072                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1073
1074                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1075                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1076                                         x86_64_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1077                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1078
1079                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1080                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1081                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1082                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1083
1084                                 } else {
1085                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1086                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1087                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1088                                 }
1089
1090                         } else {
1091                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1092                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1093                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1094
1095                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1096                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1097                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1098
1099                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1100                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1101                                         x86_64_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1102
1103                                 } else {
1104                                         if (src->regoff == iptr->dst->regoff) {
1105                                                 x86_64_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1106
1107                                         } else {
1108                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1109                                                 x86_64_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1110                                         }
1111                                 }
1112                         }
1113                         break;
1114
1115                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1116                                       /* val.i = constant                             */
1117
1118                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1119                         if (iptr->dst->flags & INMEMORY) {
1120                                 if (src->flags & INMEMORY) {
1121                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1122                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1123
1124                                 } else {
1125                                         x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1126                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1127                                 }
1128
1129                         } else {
1130                                 if (src->flags & INMEMORY) {
1131                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1132
1133                                 } else {
1134                                         if (iptr->val.i == 2) {
1135                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1136                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1137
1138                                         } else {
1139                                                 x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1140                                         }
1141                                 }
1142                         }
1143                         break;
1144
1145                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1146
1147                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1148                         if (iptr->dst->flags & INMEMORY) {
1149                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1150                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1151                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1152                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1153
1154                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1155                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1156                                         x86_64_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1157                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1158
1159                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1160                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1161                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1162                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1163
1164                                 } else {
1165                                         x86_64_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1166                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1167                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1168                                 }
1169
1170                         } else {
1171                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1172                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1173                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1174
1175                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1176                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1177                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1178
1179                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1180                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1181                                         x86_64_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1182
1183                                 } else {
1184                                         if (src->regoff == iptr->dst->regoff) {
1185                                                 x86_64_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1186
1187                                         } else {
1188                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1189                                                 x86_64_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1190                                         }
1191                                 }
1192                         }
1193                         break;
1194
1195                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1196                                       /* val.l = constant                             */
1197
1198                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1199                         if (iptr->dst->flags & INMEMORY) {
1200                                 if (src->flags & INMEMORY) {
1201                                         if (IS_IMM32(iptr->val.l)) {
1202                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1203
1204                                         } else {
1205                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1206                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1207                                         }
1208                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1209
1210                                 } else {
1211                                         if (IS_IMM32(iptr->val.l)) {
1212                                                 x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1213
1214                                         } else {
1215                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1216                                                 x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1217                                         }
1218                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1219                                 }
1220
1221                         } else {
1222                                 if (src->flags & INMEMORY) {
1223                                         if (IS_IMM32(iptr->val.l)) {
1224                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1225
1226                                         } else {
1227                                                 x86_64_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1228                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1229                                         }
1230
1231                                 } else {
1232                                         /* should match in many cases */
1233                                         if (iptr->val.l == 2) {
1234                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1235                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1236
1237                                         } else {
1238                                                 if (IS_IMM32(iptr->val.l)) {
1239                                                         x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1240
1241                                                 } else {
1242                                                         x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1243                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1244                                                         x86_64_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1245                                                 }
1246                                         }
1247                                 }
1248                         }
1249                         break;
1250
1251                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1252
1253                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1254                 if (src->prev->flags & INMEMORY) {
1255                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1256
1257                         } else {
1258                                 M_INTMOVE(src->prev->regoff, RAX);
1259                         }
1260
1261                         if (src->flags & INMEMORY) {
1262                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1263
1264                         } else {
1265                                 M_INTMOVE(src->regoff, REG_ITMP3);
1266                         }
1267                         gen_div_check(src);
1268
1269                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1270                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1271                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1272                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                /* 6 bytes */
1273
1274                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1275                         x86_64_cltd(cd);
1276                         x86_64_idivl_reg(cd, REG_ITMP3);
1277
1278                         if (iptr->dst->flags & INMEMORY) {
1279                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1280                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1281
1282                         } else {
1283                                 M_INTMOVE(RAX, iptr->dst->regoff);
1284
1285                                 if (iptr->dst->regoff != RDX) {
1286                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1287                                 }
1288                         }
1289                         break;
1290
1291                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1292                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1293                         if (src->prev->flags & INMEMORY) {
1294                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1295
1296                         } else {
1297                                 M_INTMOVE(src->prev->regoff, RAX);
1298                         }
1299
1300                         if (src->flags & INMEMORY) {
1301                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1302
1303                         } else {
1304                                 M_INTMOVE(src->regoff, REG_ITMP3);
1305                         }
1306                         gen_div_check(src);
1307
1308                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1309
1310                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1311                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1312
1313
1314                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1315                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1316                         x86_64_jcc(cd, X86_64_CC_E, 1 + 3);                    /* 6 bytes */
1317
1318                         x86_64_cltd(cd);
1319                         x86_64_idivl_reg(cd, REG_ITMP3);
1320
1321                         if (iptr->dst->flags & INMEMORY) {
1322                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1323                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1324
1325                         } else {
1326                                 M_INTMOVE(RDX, iptr->dst->regoff);
1327
1328                                 if (iptr->dst->regoff != RDX) {
1329                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1330                                 }
1331                         }
1332                         break;
1333
1334                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1335                                       /* val.i = constant                             */
1336
1337                         var_to_reg_int(s1, src, REG_ITMP1);
1338                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1339                         M_INTMOVE(s1, REG_ITMP1);
1340                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1341                         x86_64_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1342                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1343                         x86_64_shiftl_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1344                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1345                         store_reg_to_var_int(iptr->dst, d);
1346                         break;
1347
1348                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1349                                       /* val.i = constant                             */
1350
1351                         var_to_reg_int(s1, src, REG_ITMP1);
1352                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1353                         M_INTMOVE(s1, REG_ITMP1);
1354                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1355                         x86_64_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1356                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1357                         x86_64_alul_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1358                         x86_64_alul_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1359                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1360                         store_reg_to_var_int(iptr->dst, d);
1361                         break;
1362
1363
1364                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1365
1366                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1367
1368                 if (src->prev->flags & INMEMORY) {
1369                                 M_LLD(RAX, REG_SP, src->prev->regoff * 8);
1370
1371                         } else {
1372                                 M_INTMOVE(src->prev->regoff, RAX);
1373                         }
1374
1375                         if (src->flags & INMEMORY) {
1376                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1377
1378                         } else {
1379                                 M_INTMOVE(src->regoff, REG_ITMP3);
1380                         }
1381                         gen_div_check(src);
1382
1383                         /* check as described in jvm spec */
1384                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1385                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, RAX);
1386                         M_BNE(4 + 6);
1387                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1388                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
1389
1390                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1391                         x86_64_cqto(cd);
1392                         x86_64_idiv_reg(cd, REG_ITMP3);
1393
1394                         if (iptr->dst->flags & INMEMORY) {
1395                                 M_LST(RAX, REG_SP, iptr->dst->regoff * 8);
1396                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1397
1398                         } else {
1399                                 M_INTMOVE(RAX, iptr->dst->regoff);
1400
1401                                 if (iptr->dst->regoff != RDX) {
1402                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1403                                 }
1404                         }
1405                         break;
1406
1407                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1408
1409                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1410                         if (src->prev->flags & INMEMORY) {
1411                                 M_LLD(REG_ITMP1, REG_SP, src->prev->regoff * 8);
1412
1413                         } else {
1414                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1415                         }
1416
1417                         if (src->flags & INMEMORY) {
1418                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1419
1420                         } else {
1421                                 M_INTMOVE(src->regoff, REG_ITMP3);
1422                         }
1423                         gen_div_check(src);
1424
1425                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1426
1427                         /* check as described in jvm spec */
1428                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1429                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP1);
1430                         M_BNE(3 + 4 + 6);
1431
1432 #if 0
1433                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1434 #endif
1435                         M_XOR(RDX, RDX);                                       /* 3 bytes */
1436                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1437                         M_BEQ(2 + 3);                                          /* 6 bytes */
1438
1439                         x86_64_cqto(cd);
1440                         x86_64_idiv_reg(cd, REG_ITMP3);
1441
1442                         if (iptr->dst->flags & INMEMORY) {
1443                                 M_LST(RDX, REG_SP, iptr->dst->regoff * 8);
1444                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1445
1446                         } else {
1447                                 M_INTMOVE(RDX, iptr->dst->regoff);
1448
1449                                 if (iptr->dst->regoff != RDX) {
1450                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1451                                 }
1452                         }
1453                         break;
1454
1455                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1456                                       /* val.i = constant                             */
1457
1458                         var_to_reg_int(s1, src, REG_ITMP1);
1459                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1460                         M_INTMOVE(s1, REG_ITMP1);
1461                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1462                         x86_64_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1463                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1464                         x86_64_shift_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1465                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1466                         store_reg_to_var_int(iptr->dst, d);
1467                         break;
1468
1469                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1470                                       /* val.l = constant                             */
1471
1472                         var_to_reg_int(s1, src, REG_ITMP1);
1473                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1474                         M_INTMOVE(s1, REG_ITMP1);
1475                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1476                         x86_64_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1477                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1478                         x86_64_alu_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1479                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1480                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1481                         store_reg_to_var_int(iptr->dst, d);
1482                         break;
1483
1484                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1485
1486                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1487                         x86_64_emit_ishift(cd, X86_64_SHL, src, iptr);
1488                         break;
1489
1490                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1491                                       /* val.i = constant                             */
1492
1493                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1494                         x86_64_emit_ishiftconst(cd, X86_64_SHL, src, iptr);
1495                         break;
1496
1497                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1498
1499                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1500                         x86_64_emit_ishift(cd, X86_64_SAR, src, iptr);
1501                         break;
1502
1503                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1504                                       /* val.i = constant                             */
1505
1506                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1507                         x86_64_emit_ishiftconst(cd, X86_64_SAR, src, iptr);
1508                         break;
1509
1510                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1511
1512                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1513                         x86_64_emit_ishift(cd, X86_64_SHR, src, iptr);
1514                         break;
1515
1516                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1517                                       /* val.i = constant                             */
1518
1519                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1520                         x86_64_emit_ishiftconst(cd, X86_64_SHR, src, iptr);
1521                         break;
1522
1523                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1524
1525                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1526                         x86_64_emit_lshift(cd, X86_64_SHL, src, iptr);
1527                         break;
1528
1529         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1530                                           /* val.i = constant                             */
1531
1532                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1533                         x86_64_emit_lshiftconst(cd, X86_64_SHL, src, iptr);
1534                         break;
1535
1536                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1537
1538                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1539                         x86_64_emit_lshift(cd, X86_64_SAR, src, iptr);
1540                         break;
1541
1542                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1543                                       /* val.i = constant                             */
1544
1545                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1546                         x86_64_emit_lshiftconst(cd, X86_64_SAR, src, iptr);
1547                         break;
1548
1549                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1550
1551                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1552                         x86_64_emit_lshift(cd, X86_64_SHR, src, iptr);
1553                         break;
1554
1555                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1556                                       /* val.l = constant                             */
1557
1558                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1559                         x86_64_emit_lshiftconst(cd, X86_64_SHR, src, iptr);
1560                         break;
1561
1562                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1563
1564                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1565                         x86_64_emit_ialu(cd, X86_64_AND, src, iptr);
1566                         break;
1567
1568                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1569                                       /* val.i = constant                             */
1570
1571                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1572                         x86_64_emit_ialuconst(cd, X86_64_AND, src, iptr);
1573                         break;
1574
1575                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1576
1577                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1578                         x86_64_emit_lalu(cd, X86_64_AND, src, iptr);
1579                         break;
1580
1581                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1582                                       /* val.l = constant                             */
1583
1584                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1585                         x86_64_emit_laluconst(cd, X86_64_AND, src, iptr);
1586                         break;
1587
1588                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1589
1590                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1591                         x86_64_emit_ialu(cd, X86_64_OR, src, iptr);
1592                         break;
1593
1594                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1595                                       /* val.i = constant                             */
1596
1597                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1598                         x86_64_emit_ialuconst(cd, X86_64_OR, src, iptr);
1599                         break;
1600
1601                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1602
1603                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1604                         x86_64_emit_lalu(cd, X86_64_OR, src, iptr);
1605                         break;
1606
1607                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1608                                       /* val.l = constant                             */
1609
1610                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1611                         x86_64_emit_laluconst(cd, X86_64_OR, src, iptr);
1612                         break;
1613
1614                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1615
1616                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1617                         x86_64_emit_ialu(cd, X86_64_XOR, src, iptr);
1618                         break;
1619
1620                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1621                                       /* val.i = constant                             */
1622
1623                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1624                         x86_64_emit_ialuconst(cd, X86_64_XOR, src, iptr);
1625                         break;
1626
1627                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1628
1629                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1630                         x86_64_emit_lalu(cd, X86_64_XOR, src, iptr);
1631                         break;
1632
1633                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1634                                       /* val.l = constant                             */
1635
1636                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1637                         x86_64_emit_laluconst(cd, X86_64_XOR, src, iptr);
1638                         break;
1639
1640
1641                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1642                                       /* op1 = variable, val.i = constant             */
1643
1644                         /* using inc and dec is definitely faster than add -- tested      */
1645                         /* with sieve                                                     */
1646
1647                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1648                         d = var->regoff;
1649                         if (var->flags & INMEMORY) {
1650                                 if (iptr->val.i == 1) {
1651                                         x86_64_incl_membase(cd, REG_SP, d * 8);
1652
1653                                 } else if (iptr->val.i == -1) {
1654                                         x86_64_decl_membase(cd, REG_SP, d * 8);
1655
1656                                 } else {
1657                                         x86_64_alul_imm_membase(cd, X86_64_ADD, iptr->val.i, REG_SP, d * 8);
1658                                 }
1659
1660                         } else {
1661                                 if (iptr->val.i == 1) {
1662                                         x86_64_incl_reg(cd, d);
1663
1664                                 } else if (iptr->val.i == -1) {
1665                                         x86_64_decl_reg(cd, d);
1666
1667                                 } else {
1668                                         x86_64_alul_imm_reg(cd, X86_64_ADD, iptr->val.i, d);
1669                                 }
1670                         }
1671                         break;
1672
1673
1674                 /* floating operations ************************************************/
1675
1676                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1677
1678                         var_to_reg_flt(s1, src, REG_FTMP1);
1679                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1680                         disp = dseg_adds4(cd, 0x80000000);
1681                         M_FLTMOVE(s1, d);
1682                         x86_64_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1683                         x86_64_xorps_reg_reg(cd, REG_FTMP2, d);
1684                         store_reg_to_var_flt(iptr->dst, d);
1685                         break;
1686
1687                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1688
1689                         var_to_reg_flt(s1, src, REG_FTMP1);
1690                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1691                         disp = dseg_adds8(cd, 0x8000000000000000);
1692                         M_FLTMOVE(s1, d);
1693                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1694                         x86_64_xorpd_reg_reg(cd, REG_FTMP2, d);
1695                         store_reg_to_var_flt(iptr->dst, d);
1696                         break;
1697
1698                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1699
1700                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1701                         var_to_reg_flt(s2, src, REG_FTMP2);
1702                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1703                         if (s1 == d) {
1704                                 x86_64_addss_reg_reg(cd, s2, d);
1705                         } else if (s2 == d) {
1706                                 x86_64_addss_reg_reg(cd, s1, d);
1707                         } else {
1708                                 M_FLTMOVE(s1, d);
1709                                 x86_64_addss_reg_reg(cd, s2, d);
1710                         }
1711                         store_reg_to_var_flt(iptr->dst, d);
1712                         break;
1713
1714                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1715
1716                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1717                         var_to_reg_flt(s2, src, REG_FTMP2);
1718                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1719                         if (s1 == d) {
1720                                 x86_64_addsd_reg_reg(cd, s2, d);
1721                         } else if (s2 == d) {
1722                                 x86_64_addsd_reg_reg(cd, s1, d);
1723                         } else {
1724                                 M_FLTMOVE(s1, d);
1725                                 x86_64_addsd_reg_reg(cd, s2, d);
1726                         }
1727                         store_reg_to_var_flt(iptr->dst, d);
1728                         break;
1729
1730                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1731
1732                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1733                         var_to_reg_flt(s2, src, REG_FTMP2);
1734                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1735                         if (s2 == d) {
1736                                 M_FLTMOVE(s2, REG_FTMP2);
1737                                 s2 = REG_FTMP2;
1738                         }
1739                         M_FLTMOVE(s1, d);
1740                         x86_64_subss_reg_reg(cd, s2, d);
1741                         store_reg_to_var_flt(iptr->dst, d);
1742                         break;
1743
1744                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1745
1746                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1747                         var_to_reg_flt(s2, src, REG_FTMP2);
1748                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1749                         if (s2 == d) {
1750                                 M_FLTMOVE(s2, REG_FTMP2);
1751                                 s2 = REG_FTMP2;
1752                         }
1753                         M_FLTMOVE(s1, d);
1754                         x86_64_subsd_reg_reg(cd, s2, d);
1755                         store_reg_to_var_flt(iptr->dst, d);
1756                         break;
1757
1758                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1759
1760                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1761                         var_to_reg_flt(s2, src, REG_FTMP2);
1762                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1763                         if (s1 == d) {
1764                                 x86_64_mulss_reg_reg(cd, s2, d);
1765                         } else if (s2 == d) {
1766                                 x86_64_mulss_reg_reg(cd, s1, d);
1767                         } else {
1768                                 M_FLTMOVE(s1, d);
1769                                 x86_64_mulss_reg_reg(cd, s2, d);
1770                         }
1771                         store_reg_to_var_flt(iptr->dst, d);
1772                         break;
1773
1774                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1775
1776                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1777                         var_to_reg_flt(s2, src, REG_FTMP2);
1778                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1779                         if (s1 == d) {
1780                                 x86_64_mulsd_reg_reg(cd, s2, d);
1781                         } else if (s2 == d) {
1782                                 x86_64_mulsd_reg_reg(cd, s1, d);
1783                         } else {
1784                                 M_FLTMOVE(s1, d);
1785                                 x86_64_mulsd_reg_reg(cd, s2, d);
1786                         }
1787                         store_reg_to_var_flt(iptr->dst, d);
1788                         break;
1789
1790                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1791
1792                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1793                         var_to_reg_flt(s2, src, REG_FTMP2);
1794                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1795                         if (s2 == d) {
1796                                 M_FLTMOVE(s2, REG_FTMP2);
1797                                 s2 = REG_FTMP2;
1798                         }
1799                         M_FLTMOVE(s1, d);
1800                         x86_64_divss_reg_reg(cd, s2, d);
1801                         store_reg_to_var_flt(iptr->dst, d);
1802                         break;
1803
1804                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1805
1806                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1807                         var_to_reg_flt(s2, src, REG_FTMP2);
1808                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1809                         if (s2 == d) {
1810                                 M_FLTMOVE(s2, REG_FTMP2);
1811                                 s2 = REG_FTMP2;
1812                         }
1813                         M_FLTMOVE(s1, d);
1814                         x86_64_divsd_reg_reg(cd, s2, d);
1815                         store_reg_to_var_flt(iptr->dst, d);
1816                         break;
1817
1818                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1819
1820                         var_to_reg_int(s1, src, REG_ITMP1);
1821                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1822                         x86_64_cvtsi2ss_reg_reg(cd, s1, d);
1823                         store_reg_to_var_flt(iptr->dst, d);
1824                         break;
1825
1826                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1827
1828                         var_to_reg_int(s1, src, REG_ITMP1);
1829                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1830                         x86_64_cvtsi2sd_reg_reg(cd, s1, d);
1831                         store_reg_to_var_flt(iptr->dst, d);
1832                         break;
1833
1834                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1835
1836                         var_to_reg_int(s1, src, REG_ITMP1);
1837                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1838                         x86_64_cvtsi2ssq_reg_reg(cd, s1, d);
1839                         store_reg_to_var_flt(iptr->dst, d);
1840                         break;
1841
1842                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1843
1844                         var_to_reg_int(s1, src, REG_ITMP1);
1845                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1846                         x86_64_cvtsi2sdq_reg_reg(cd, s1, d);
1847                         store_reg_to_var_flt(iptr->dst, d);
1848                         break;
1849
1850                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1851
1852                         var_to_reg_flt(s1, src, REG_FTMP1);
1853                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1854                         x86_64_cvttss2si_reg_reg(cd, s1, d);
1855                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1856                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1857                         x86_64_jcc(cd, X86_64_CC_NE, a);
1858                         M_FLTMOVE(s1, REG_FTMP1);
1859                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_f2i, REG_ITMP2);
1860                         x86_64_call_reg(cd, REG_ITMP2);
1861                         M_INTMOVE(REG_RESULT, d);
1862                         store_reg_to_var_int(iptr->dst, d);
1863                         break;
1864
1865                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1866
1867                         var_to_reg_flt(s1, src, REG_FTMP1);
1868                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1869                         x86_64_cvttsd2si_reg_reg(cd, s1, d);
1870                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1871                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1872                         x86_64_jcc(cd, X86_64_CC_NE, a);
1873                         M_FLTMOVE(s1, REG_FTMP1);
1874                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_d2i, REG_ITMP2);
1875                         x86_64_call_reg(cd, REG_ITMP2);
1876                         M_INTMOVE(REG_RESULT, d);
1877                         store_reg_to_var_int(iptr->dst, d);
1878                         break;
1879
1880                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1881
1882                         var_to_reg_flt(s1, src, REG_FTMP1);
1883                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1884                         x86_64_cvttss2siq_reg_reg(cd, s1, d);
1885                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1886                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1887                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1888                         x86_64_jcc(cd, X86_64_CC_NE, a);
1889                         M_FLTMOVE(s1, REG_FTMP1);
1890                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_f2l, REG_ITMP2);
1891                         x86_64_call_reg(cd, REG_ITMP2);
1892                         M_INTMOVE(REG_RESULT, d);
1893                         store_reg_to_var_int(iptr->dst, d);
1894                         break;
1895
1896                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1897
1898                         var_to_reg_flt(s1, src, REG_FTMP1);
1899                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1900                         x86_64_cvttsd2siq_reg_reg(cd, s1, d);
1901                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1902                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1903                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1904                         x86_64_jcc(cd, X86_64_CC_NE, a);
1905                         M_FLTMOVE(s1, REG_FTMP1);
1906                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_d2l, REG_ITMP2);
1907                         x86_64_call_reg(cd, REG_ITMP2);
1908                         M_INTMOVE(REG_RESULT, d);
1909                         store_reg_to_var_int(iptr->dst, d);
1910                         break;
1911
1912                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1913
1914                         var_to_reg_flt(s1, src, REG_FTMP1);
1915                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1916                         x86_64_cvtss2sd_reg_reg(cd, s1, d);
1917                         store_reg_to_var_flt(iptr->dst, d);
1918                         break;
1919
1920                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1921
1922                         var_to_reg_flt(s1, src, REG_FTMP1);
1923                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1924                         x86_64_cvtsd2ss_reg_reg(cd, s1, d);
1925                         store_reg_to_var_flt(iptr->dst, d);
1926                         break;
1927
1928                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1929                                           /* == => 0, < => 1, > => -1 */
1930
1931                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1932                         var_to_reg_flt(s2, src, REG_FTMP2);
1933                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1934                         M_CLR(d);
1935                         M_MOV_IMM(1, REG_ITMP1);
1936                         M_MOV_IMM(-1, REG_ITMP2);
1937                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1938                         M_CMOVB(REG_ITMP1, d);
1939                         M_CMOVA(REG_ITMP2, d);
1940                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1941                         store_reg_to_var_int(iptr->dst, d);
1942                         break;
1943
1944                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1945                                           /* == => 0, < => 1, > => -1 */
1946
1947                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1948                         var_to_reg_flt(s2, src, REG_FTMP2);
1949                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1950                         M_CLR(d);
1951                         M_MOV_IMM(1, REG_ITMP1);
1952                         M_MOV_IMM(-1, REG_ITMP2);
1953                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1954                         M_CMOVB(REG_ITMP1, d);
1955                         M_CMOVA(REG_ITMP2, d);
1956                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1957                         store_reg_to_var_int(iptr->dst, d);
1958                         break;
1959
1960                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1961                                           /* == => 0, < => 1, > => -1 */
1962
1963                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1964                         var_to_reg_flt(s2, src, REG_FTMP2);
1965                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1966                         M_CLR(d);
1967                         M_MOV_IMM(1, REG_ITMP1);
1968                         M_MOV_IMM(-1, REG_ITMP2);
1969                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1970                         M_CMOVB(REG_ITMP1, d);
1971                         M_CMOVA(REG_ITMP2, d);
1972                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1973                         store_reg_to_var_int(iptr->dst, d);
1974                         break;
1975
1976                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1977                                           /* == => 0, < => 1, > => -1 */
1978
1979                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1980                         var_to_reg_flt(s2, src, REG_FTMP2);
1981                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1982                         M_CLR(d);
1983                         M_MOV_IMM(1, REG_ITMP1);
1984                         M_MOV_IMM(-1, REG_ITMP2);
1985                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1986                         M_CMOVB(REG_ITMP1, d);
1987                         M_CMOVA(REG_ITMP2, d);
1988                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1989                         store_reg_to_var_int(iptr->dst, d);
1990                         break;
1991
1992
1993                 /* memory operations **************************************************/
1994
1995                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1996
1997                         var_to_reg_int(s1, src, REG_ITMP1);
1998                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1999                         gen_nullptr_check(s1);
2000                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
2001                         store_reg_to_var_int(iptr->dst, d);
2002                         break;
2003
2004                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2005
2006                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2007                         var_to_reg_int(s2, src, REG_ITMP2);
2008                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2009                         if (iptr->op1 == 0) {
2010                                 gen_nullptr_check(s1);
2011                                 gen_bound_check;
2012                         }
2013                         x86_64_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2014                         store_reg_to_var_int(iptr->dst, d);
2015                         break;
2016
2017                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2018
2019                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2020                         var_to_reg_int(s2, src, REG_ITMP2);
2021                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2022                         if (iptr->op1 == 0) {
2023                                 gen_nullptr_check(s1);
2024                                 gen_bound_check;
2025                         }
2026                         x86_64_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2027                         store_reg_to_var_int(iptr->dst, d);
2028                         break;
2029
2030                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2031
2032                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2033                         var_to_reg_int(s2, src, REG_ITMP2);
2034                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2035                         if (iptr->op1 == 0) {
2036                                 gen_nullptr_check(s1);
2037                                 gen_bound_check;
2038                         }
2039                         x86_64_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2040                         store_reg_to_var_int(iptr->dst, d);
2041                         break;
2042
2043                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
2044
2045                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2046                         var_to_reg_int(s2, src, REG_ITMP2);
2047                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2048                         if (iptr->op1 == 0) {
2049                                 gen_nullptr_check(s1);
2050                                 gen_bound_check;
2051                         }
2052                         x86_64_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
2053                         store_reg_to_var_int(iptr->dst, d);
2054                         break;
2055
2056                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
2057
2058                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2059                         var_to_reg_int(s2, src, REG_ITMP2);
2060                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2061                         if (iptr->op1 == 0) {
2062                                 gen_nullptr_check(s1);
2063                                 gen_bound_check;
2064                         }
2065                         x86_64_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
2066                         store_reg_to_var_int(iptr->dst, d);
2067                         break;
2068
2069                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
2070
2071                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2072                         var_to_reg_int(s2, src, REG_ITMP2);
2073                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2074                         if (iptr->op1 == 0) {
2075                                 gen_nullptr_check(s1);
2076                                 gen_bound_check;
2077                         }
2078                         x86_64_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2079                         store_reg_to_var_flt(iptr->dst, d);
2080                         break;
2081
2082                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2083
2084                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2085                         var_to_reg_int(s2, src, REG_ITMP2);
2086                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2087                         if (iptr->op1 == 0) {
2088                                 gen_nullptr_check(s1);
2089                                 gen_bound_check;
2090                         }
2091                         x86_64_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2092                         store_reg_to_var_flt(iptr->dst, d);
2093                         break;
2094
2095                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
2096
2097                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2098                         var_to_reg_int(s2, src, REG_ITMP2);
2099                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2100                         if (iptr->op1 == 0) {
2101                                 gen_nullptr_check(s1);
2102                                 gen_bound_check;
2103                         }
2104                         x86_64_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
2105                         store_reg_to_var_int(iptr->dst, d);
2106                         break;
2107
2108
2109                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2110
2111                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2112                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2113                         if (iptr->op1 == 0) {
2114                                 gen_nullptr_check(s1);
2115                                 gen_bound_check;
2116                         }
2117                         var_to_reg_int(s3, src, REG_ITMP3);
2118                         x86_64_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2119                         break;
2120
2121                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2122
2123                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2124                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2125                         if (iptr->op1 == 0) {
2126                                 gen_nullptr_check(s1);
2127                                 gen_bound_check;
2128                         }
2129                         var_to_reg_int(s3, src, REG_ITMP3);
2130                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2131                         break;
2132
2133                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2134
2135                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2136                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2137                         if (iptr->op1 == 0) {
2138                                 gen_nullptr_check(s1);
2139                                 gen_bound_check;
2140                         }
2141                         var_to_reg_int(s3, src, REG_ITMP3);
2142                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2143                         break;
2144
2145                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2146
2147                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2148                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2149                         if (iptr->op1 == 0) {
2150                                 gen_nullptr_check(s1);
2151                                 gen_bound_check;
2152                         }
2153                         var_to_reg_int(s3, src, REG_ITMP3);
2154                         x86_64_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2155                         break;
2156
2157                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2158
2159                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2160                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2161                         if (iptr->op1 == 0) {
2162                                 gen_nullptr_check(s1);
2163                                 gen_bound_check;
2164                         }
2165                         var_to_reg_int(s3, src, REG_ITMP3);
2166                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2167                         break;
2168
2169                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2170
2171                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2172                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2173                         if (iptr->op1 == 0) {
2174                                 gen_nullptr_check(s1);
2175                                 gen_bound_check;
2176                         }
2177                         var_to_reg_flt(s3, src, REG_FTMP3);
2178                         x86_64_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2179                         break;
2180
2181                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2182
2183                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2184                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2185                         if (iptr->op1 == 0) {
2186                                 gen_nullptr_check(s1);
2187                                 gen_bound_check;
2188                         }
2189                         var_to_reg_flt(s3, src, REG_FTMP3);
2190                         x86_64_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2191                         break;
2192
2193                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2194
2195                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2196                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2197                         if (iptr->op1 == 0) {
2198                                 gen_nullptr_check(s1);
2199                                 gen_bound_check;
2200                         }
2201                         var_to_reg_int(s3, src, REG_ITMP3);
2202
2203                         M_MOV(s1, rd->argintregs[0]);
2204                         M_MOV(s3, rd->argintregs[1]);
2205                         M_MOV_IMM((ptrint) BUILTIN_canstore, REG_ITMP1);
2206                         M_CALL(REG_ITMP1);
2207                         M_TEST(REG_RESULT);
2208                         M_BEQ(0);
2209                         codegen_addxstorerefs(cd, cd->mcodeptr);
2210
2211                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2212                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2213                         var_to_reg_int(s3, src, REG_ITMP3);
2214                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2215                         break;
2216
2217
2218                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2219
2220                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2221                         var_to_reg_int(s2, src, REG_ITMP2);
2222                         if (iptr->op1 == 0) {
2223                                 gen_nullptr_check(s1);
2224                                 gen_bound_check;
2225                         }
2226                         x86_64_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2227                         break;
2228
2229                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2230
2231                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2232                         var_to_reg_int(s2, src, REG_ITMP2);
2233                         if (iptr->op1 == 0) {
2234                                 gen_nullptr_check(s1);
2235                                 gen_bound_check;
2236                         }
2237                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2238                         break;
2239
2240                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2241
2242                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2243                         var_to_reg_int(s2, src, REG_ITMP2);
2244                         if (iptr->op1 == 0) {
2245                                 gen_nullptr_check(s1);
2246                                 gen_bound_check;
2247                         }
2248                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2249                         break;
2250
2251                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2252
2253                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2254                         var_to_reg_int(s2, src, REG_ITMP2);
2255                         if (iptr->op1 == 0) {
2256                                 gen_nullptr_check(s1);
2257                                 gen_bound_check;
2258                         }
2259                         x86_64_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2260                         break;
2261
2262                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2263
2264                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2265                         var_to_reg_int(s2, src, REG_ITMP2);
2266                         if (iptr->op1 == 0) {
2267                                 gen_nullptr_check(s1);
2268                                 gen_bound_check;
2269                         }
2270
2271                         if (IS_IMM32(iptr->val.l)) {
2272                                 x86_64_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2273                         } else {
2274                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2275                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2276                         }
2277                         break;
2278
2279                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2280
2281                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2282                         var_to_reg_int(s2, src, REG_ITMP2);
2283                         if (iptr->op1 == 0) {
2284                                 gen_nullptr_check(s1);
2285                                 gen_bound_check;
2286                         }
2287                         x86_64_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2288                         break;
2289
2290
2291                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2292                                       /* op1 = type, val.a = field address            */
2293
2294                         if (iptr->val.a == NULL) {
2295                                 disp = dseg_addaddress(cd, NULL);
2296
2297 /*                              PROFILE_CYCLE_STOP; */
2298
2299                                 codegen_addpatchref(cd, cd->mcodeptr,
2300                                                                         PATCHER_get_putstatic,
2301                                                                         (unresolved_field *) iptr->target, disp);
2302
2303                                 if (opt_showdisassemble) {
2304                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2305                                 }
2306
2307 /*                              PROFILE_CYCLE_START; */
2308
2309                         } else {
2310                                 fieldinfo *fi = iptr->val.a;
2311
2312                                 disp = dseg_addaddress(cd, &(fi->value));
2313
2314                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2315                                         PROFILE_CYCLE_STOP;
2316
2317                                         codegen_addpatchref(cd, cd->mcodeptr,
2318                                                                                 PATCHER_clinit, fi->class, 0);
2319
2320                                         if (opt_showdisassemble) {
2321                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2322                                         }
2323
2324                                         PROFILE_CYCLE_START;
2325                                 }
2326                         }
2327
2328                         /* This approach is much faster than moving the field
2329                            address inline into a register. */
2330
2331                         M_ALD(REG_ITMP2, RIP, -(((ptrint) cd->mcodeptr + 7) -
2332                                                                         (ptrint) cd->mcodebase) + disp);
2333
2334                         switch (iptr->op1) {
2335                         case TYPE_INT:
2336                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2337                                 M_ILD(d, REG_ITMP2, 0);
2338                                 store_reg_to_var_int(iptr->dst, d);
2339                                 break;
2340                         case TYPE_LNG:
2341                         case TYPE_ADR:
2342                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2343                                 M_LLD(d, REG_ITMP2, 0);
2344                                 store_reg_to_var_int(iptr->dst, d);
2345                                 break;
2346                         case TYPE_FLT:
2347                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2348                                 x86_64_movss_membase_reg(cd, REG_ITMP2, 0, d);
2349                                 store_reg_to_var_flt(iptr->dst, d);
2350                                 break;
2351                         case TYPE_DBL:
2352                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2353                                 x86_64_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2354                                 store_reg_to_var_flt(iptr->dst, d);
2355                                 break;
2356                         }
2357                         break;
2358
2359                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2360                                       /* op1 = type, val.a = field address            */
2361
2362                         if (iptr->val.a == NULL) {
2363                                 disp = dseg_addaddress(cd, NULL);
2364
2365 /*                              PROFILE_CYCLE_STOP; */
2366
2367                                 codegen_addpatchref(cd, cd->mcodeptr,
2368                                                                         PATCHER_get_putstatic,
2369                                                                         (unresolved_field *) iptr->target, disp);
2370
2371                                 if (opt_showdisassemble) {
2372                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2373                                 }
2374
2375 /*                              PROFILE_CYCLE_START; */
2376
2377                         } else {
2378                                 fieldinfo *fi = iptr->val.a;
2379
2380                                 disp = dseg_addaddress(cd, &(fi->value));
2381
2382                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2383                                         PROFILE_CYCLE_STOP;
2384
2385                                         codegen_addpatchref(cd, cd->mcodeptr,
2386                                                                                 PATCHER_clinit, fi->class, 0);
2387
2388                                         if (opt_showdisassemble) {
2389                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2390                                         }
2391
2392                                         PROFILE_CYCLE_START;
2393                                 }
2394                         }
2395
2396                         /* This approach is much faster than moving the field
2397                            address inline into a register. */
2398
2399                         M_ALD(REG_ITMP2, RIP, -(((ptrint) cd->mcodeptr + 7) -
2400                                                                         (ptrint) cd->mcodebase) + disp);
2401
2402                         switch (iptr->op1) {
2403                         case TYPE_INT:
2404                                 var_to_reg_int(s2, src, REG_ITMP1);
2405                                 M_IST(s2, REG_ITMP2, 0);
2406                                 break;
2407                         case TYPE_LNG:
2408                         case TYPE_ADR:
2409                                 var_to_reg_int(s2, src, REG_ITMP1);
2410                                 M_LST(s2, REG_ITMP2, 0);
2411                                 break;
2412                         case TYPE_FLT:
2413                                 var_to_reg_flt(s2, src, REG_FTMP1);
2414                                 x86_64_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2415                                 break;
2416                         case TYPE_DBL:
2417                                 var_to_reg_flt(s2, src, REG_FTMP1);
2418                                 x86_64_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2419                                 break;
2420                         }
2421                         break;
2422
2423                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2424                                           /* val = value (in current instruction)     */
2425                                           /* op1 = type, val.a = field address (in    */
2426                                           /* following NOP)                           */
2427
2428                         if (iptr[1].val.a == NULL) {
2429                                 disp = dseg_addaddress(cd, NULL);
2430
2431 /*                              PROFILE_CYCLE_STOP; */
2432
2433                                 codegen_addpatchref(cd, cd->mcodeptr,
2434                                                                         PATCHER_get_putstatic,
2435                                                                         (unresolved_field *) iptr[1].target, disp);
2436
2437                                 if (opt_showdisassemble) {
2438                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2439                                 }
2440
2441 /*                              PROFILE_CYCLE_START; */
2442
2443                         } else {
2444                                 fieldinfo *fi = iptr[1].val.a;
2445
2446                                 disp = dseg_addaddress(cd, &(fi->value));
2447
2448                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2449                                         PROFILE_CYCLE_STOP;
2450
2451                                         codegen_addpatchref(cd, cd->mcodeptr,
2452                                                                                 PATCHER_clinit, fi->class, 0);
2453
2454                                         if (opt_showdisassemble) {
2455                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2456                                         }
2457
2458                                         PROFILE_CYCLE_START;
2459                                 }
2460                         }
2461
2462                         /* This approach is much faster than moving the field
2463                            address inline into a register. */
2464
2465                         M_ALD(REG_ITMP1, RIP, -(((ptrint) cd->mcodeptr + 7) -
2466                                                                         (ptrint) cd->mcodebase) + disp);
2467
2468                         switch (iptr->op1) {
2469                         case TYPE_INT:
2470                         case TYPE_FLT:
2471                                 M_IST_IMM(iptr->val.i, REG_ITMP1, 0);
2472                                 break;
2473                         case TYPE_LNG:
2474                         case TYPE_ADR:
2475                         case TYPE_DBL:
2476                                 if (IS_IMM32(iptr->val.l)) {
2477                                         M_LST_IMM32(iptr->val.l, REG_ITMP1, 0);
2478                                 } else {
2479                                         M_IST_IMM(iptr->val.l, REG_ITMP1, 0);
2480                                         M_IST_IMM(iptr->val.l >> 32, REG_ITMP1, 4);
2481                                 }
2482                                 break;
2483                         }
2484                         break;
2485
2486                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2487                                       /* op1 = type, val.i = field offset             */
2488
2489                         var_to_reg_int(s1, src, REG_ITMP1);
2490                         gen_nullptr_check(s1);
2491
2492                         if (iptr->val.a == NULL) {
2493 /*                              PROFILE_CYCLE_STOP; */
2494
2495                                 codegen_addpatchref(cd, cd->mcodeptr,
2496                                                                         PATCHER_get_putfield,
2497                                                                         (unresolved_field *) iptr->target, 0);
2498
2499                                 if (opt_showdisassemble) {
2500                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2501                                 }
2502
2503 /*                              PROFILE_CYCLE_START; */
2504
2505                                 disp = 0;
2506
2507                         } else {
2508                                 disp = ((fieldinfo *) (iptr->val.a))->offset;
2509                         }
2510
2511                         switch (iptr->op1) {
2512                         case TYPE_INT:
2513                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2514                                 if (iptr->val.a == NULL)
2515                                         M_ILD32(d, s1, disp);
2516                                 else
2517                                         M_ILD(d, s1, disp);
2518                                 store_reg_to_var_int(iptr->dst, d);
2519                                 break;
2520                         case TYPE_LNG:
2521                         case TYPE_ADR:
2522                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2523                                 if (iptr->val.a == NULL)
2524                                         M_LLD32(d, s1, disp);
2525                                 else
2526                                         M_LLD(d, s1, disp);
2527                                 store_reg_to_var_int(iptr->dst, d);
2528                                 break;
2529                         case TYPE_FLT:
2530                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2531                                 x86_64_movss_membase32_reg(cd, s1, disp, d);
2532                                 store_reg_to_var_flt(iptr->dst, d);
2533                                 break;
2534                         case TYPE_DBL:
2535                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2536                                 x86_64_movsd_membase32_reg(cd, s1, disp, d);
2537                                 store_reg_to_var_flt(iptr->dst, d);
2538                                 break;
2539                         }
2540                         break;
2541
2542                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2543                                       /* op1 = type, val.i = field offset             */
2544
2545                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2546                         gen_nullptr_check(s1);
2547
2548                         if (IS_INT_LNG_TYPE(iptr->op1)) {
2549                                 var_to_reg_int(s2, src, REG_ITMP2);
2550                         } else {
2551                                 var_to_reg_flt(s2, src, REG_FTMP2);
2552                         }
2553
2554                         if (iptr->val.a == NULL) {
2555 /*                              PROFILE_CYCLE_STOP; */
2556
2557                                 codegen_addpatchref(cd, cd->mcodeptr,
2558                                                                         PATCHER_get_putfield,
2559                                                                         (unresolved_field *) iptr->target, 0);
2560
2561                                 if (opt_showdisassemble) {
2562                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2563                                 }
2564
2565 /*                              PROFILE_CYCLE_START; */
2566
2567                                 disp = 0;
2568
2569                         } else {
2570                                 disp = ((fieldinfo *) (iptr->val.a))->offset;
2571                         }
2572
2573                         switch (iptr->op1) {
2574                         case TYPE_INT:
2575                                 if (iptr->val.a == NULL)
2576                                         M_IST32(s2, s1, disp);
2577                                 else
2578                                         M_IST(s2, s1, disp);
2579                                 break;
2580                         case TYPE_LNG:
2581                         case TYPE_ADR:
2582                                 if (iptr->val.a == NULL)
2583                                         M_LST32(s2, s1, disp);
2584                                 else
2585                                         M_LST(s2, s1, disp);
2586                                 break;
2587                         case TYPE_FLT:
2588                                 x86_64_movss_reg_membase32(cd, s2, s1, disp);
2589                                 break;
2590                         case TYPE_DBL:
2591                                 x86_64_movsd_reg_membase32(cd, s2, s1, disp);
2592                                 break;
2593                         }
2594                         break;
2595
2596                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2597                                           /* val = value (in current instruction)     */
2598                                           /* op1 = type, val.a = field address (in    */
2599                                           /* following NOP)                           */
2600
2601                         var_to_reg_int(s1, src, REG_ITMP1);
2602                         gen_nullptr_check(s1);
2603
2604                         if (iptr[1].val.a == NULL) {
2605 /*                              PROFILE_CYCLE_STOP; */
2606
2607                                 codegen_addpatchref(cd, cd->mcodeptr,
2608                                                                         PATCHER_putfieldconst,
2609                                                                         (unresolved_field *) iptr[1].target, 0);
2610
2611                                 if (opt_showdisassemble) {
2612                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2613                                 }
2614
2615 /*                              PROFILE_CYCLE_START; */
2616
2617                                 disp = 0;
2618
2619                         } else {
2620                                 disp = ((fieldinfo *) (iptr[1].val.a))->offset;
2621                         }
2622
2623                         switch (iptr->op1) {
2624                         case TYPE_INT:
2625                         case TYPE_FLT:
2626                                 if (iptr[1].val.a == NULL)
2627                                         M_IST32_IMM(iptr->val.i, s1, disp);
2628                                 else
2629                                         M_IST_IMM(iptr->val.i, s1, disp);
2630                                 break;
2631                         case TYPE_LNG:
2632                         case TYPE_ADR:
2633                         case TYPE_DBL:
2634                                 /* We can only optimize the move, if the class is
2635                                    resolved. Otherwise we don't know what to patch. */
2636                                 if (iptr[1].val.a == NULL) {
2637                                         M_IST32_IMM(iptr->val.l, s1, disp);
2638                                         M_IST32_IMM(iptr->val.l >> 32, s1, disp + 4);
2639                                 } else {
2640                                         if (IS_IMM32(iptr->val.l)) {
2641                                                 M_LST_IMM32(iptr->val.l, s1, disp);
2642                                         } else {
2643                                                 M_IST_IMM(iptr->val.l, s1, disp);
2644                                                 M_IST_IMM(iptr->val.l >> 32, s1, disp + 4);
2645                                         }
2646                                 }
2647                                 break;
2648                         }
2649                         break;
2650
2651
2652                 /* branch operations **************************************************/
2653
2654                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2655
2656                         var_to_reg_int(s1, src, REG_ITMP1);
2657                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2658
2659                         PROFILE_CYCLE_STOP;
2660
2661 #ifdef ENABLE_VERIFIER
2662                         if (iptr->val.a) {
2663                                 codegen_addpatchref(cd, cd->mcodeptr,
2664                                                                         PATCHER_athrow_areturn,
2665                                                                         (unresolved_class *) iptr->val.a, 0);
2666
2667                                 if (opt_showdisassemble) {
2668                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2669                                 }
2670                         }
2671 #endif /* ENABLE_VERIFIER */
2672
2673                         M_CALL_IMM(0);                            /* passing exception pc */
2674                         M_POP(REG_ITMP2_XPC);
2675
2676                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
2677                         M_JMP(REG_ITMP3);
2678                         break;
2679
2680                 case ICMD_GOTO:         /* ... ==> ...                                */
2681                                         /* op1 = target JavaVM pc                     */
2682
2683                         M_JMP_IMM(0);
2684                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2685                         break;
2686
2687                 case ICMD_JSR:          /* ... ==> ...                                */
2688                                         /* op1 = target JavaVM pc                     */
2689
2690                         M_CALL_IMM(0);
2691                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2692                         break;
2693
2694                 case ICMD_RET:          /* ... ==> ...                                */
2695                                         /* op1 = local variable                       */
2696
2697                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2698                         var_to_reg_int(s1, var, REG_ITMP1);
2699                         M_JMP(s1);
2700                         break;
2701
2702                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2703                                         /* op1 = target JavaVM pc                     */
2704
2705                         if (src->flags & INMEMORY)
2706                                 M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
2707                         else
2708                                 M_TEST(src->regoff);
2709                         M_BEQ(0);
2710                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2711                         break;
2712
2713                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2714                                         /* op1 = target JavaVM pc                     */
2715
2716                         if (src->flags & INMEMORY)
2717                                 M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
2718                         else
2719                                 M_TEST(src->regoff);
2720                         M_BNE(0);
2721                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2722                         break;
2723
2724                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2725                                         /* op1 = target JavaVM pc, val.i = constant   */
2726
2727                         x86_64_emit_ifcc(cd, X86_64_CC_E, src, iptr);
2728                         break;
2729
2730                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2731                                         /* op1 = target JavaVM pc, val.i = constant   */
2732
2733                         x86_64_emit_ifcc(cd, X86_64_CC_L, src, iptr);
2734                         break;
2735
2736                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2737                                         /* op1 = target JavaVM pc, val.i = constant   */
2738
2739                         x86_64_emit_ifcc(cd, X86_64_CC_LE, src, iptr);
2740                         break;
2741
2742                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2743                                         /* op1 = target JavaVM pc, val.i = constant   */
2744
2745                         x86_64_emit_ifcc(cd, X86_64_CC_NE, src, iptr);
2746                         break;
2747
2748                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2749                                         /* op1 = target JavaVM pc, val.i = constant   */
2750
2751                         x86_64_emit_ifcc(cd, X86_64_CC_G, src, iptr);
2752                         break;
2753
2754                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2755                                         /* op1 = target JavaVM pc, val.i = constant   */
2756
2757                         x86_64_emit_ifcc(cd, X86_64_CC_GE, src, iptr);
2758                         break;
2759
2760                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2761                                         /* op1 = target JavaVM pc, val.l = constant   */
2762
2763                         x86_64_emit_if_lcc(cd, X86_64_CC_E, src, iptr);
2764                         break;
2765
2766                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2767                                         /* op1 = target JavaVM pc, val.l = constant   */
2768
2769                         x86_64_emit_if_lcc(cd, X86_64_CC_L, src, iptr);
2770                         break;
2771
2772                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2773                                         /* op1 = target JavaVM pc, val.l = constant   */
2774
2775                         x86_64_emit_if_lcc(cd, X86_64_CC_LE, src, iptr);
2776                         break;
2777
2778                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2779                                         /* op1 = target JavaVM pc, val.l = constant   */
2780
2781                         x86_64_emit_if_lcc(cd, X86_64_CC_NE, src, iptr);
2782                         break;
2783
2784                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2785                                         /* op1 = target JavaVM pc, val.l = constant   */
2786
2787                         x86_64_emit_if_lcc(cd, X86_64_CC_G, src, iptr);
2788                         break;
2789
2790                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2791                                         /* op1 = target JavaVM pc, val.l = constant   */
2792
2793                         x86_64_emit_if_lcc(cd, X86_64_CC_GE, src, iptr);
2794                         break;
2795
2796                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2797                                         /* op1 = target JavaVM pc                     */
2798
2799                         x86_64_emit_if_icmpcc(cd, X86_64_CC_E, src, iptr);
2800                         break;
2801
2802                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2803                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2804
2805                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_E, src, iptr);
2806                         break;
2807
2808                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2809                                         /* op1 = target JavaVM pc                     */
2810
2811                         x86_64_emit_if_icmpcc(cd, X86_64_CC_NE, src, iptr);
2812                         break;
2813
2814                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2815                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2816
2817                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_NE, src, iptr);
2818                         break;
2819
2820                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2821                                         /* op1 = target JavaVM pc                     */
2822
2823                         x86_64_emit_if_icmpcc(cd, X86_64_CC_L, src, iptr);
2824                         break;
2825
2826                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2827                                     /* op1 = target JavaVM pc                     */
2828
2829                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_L, src, iptr);
2830                         break;
2831
2832                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2833                                         /* op1 = target JavaVM pc                     */
2834
2835                         x86_64_emit_if_icmpcc(cd, X86_64_CC_G, src, iptr);
2836                         break;
2837
2838                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2839                                 /* op1 = target JavaVM pc                     */
2840
2841                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_G, src, iptr);
2842                         break;
2843
2844                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2845                                         /* op1 = target JavaVM pc                     */
2846
2847                         x86_64_emit_if_icmpcc(cd, X86_64_CC_LE, src, iptr);
2848                         break;
2849
2850                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2851                                         /* op1 = target JavaVM pc                     */
2852
2853                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_LE, src, iptr);
2854                         break;
2855
2856                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2857                                         /* op1 = target JavaVM pc                     */
2858
2859                         x86_64_emit_if_icmpcc(cd, X86_64_CC_GE, src, iptr);
2860                         break;
2861
2862                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2863                                     /* op1 = target JavaVM pc                     */
2864
2865                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_GE, src, iptr);
2866                         break;
2867
2868                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2869
2870                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2871                         break;
2872
2873                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2874                 case ICMD_IFNE_ICONST:  /* val.i = constant                           */
2875                 case ICMD_IFLT_ICONST:
2876                 case ICMD_IFGE_ICONST:
2877                 case ICMD_IFGT_ICONST:
2878                 case ICMD_IFLE_ICONST:
2879
2880                         var_to_reg_int(s1, src, REG_ITMP1);
2881                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2882                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2883                                 if (s1 == d) {
2884                                         M_INTMOVE(s1, REG_ITMP1);
2885                                         s1 = REG_ITMP1;
2886                                 }
2887                                 if (iptr[1].val.i == 0)
2888                                         M_CLR(d);
2889                                 else
2890                                         M_IMOV_IMM(iptr[1].val.i, d);
2891                         }
2892                         if (iptr->val.i == 0)
2893                                 M_CLR(REG_ITMP2);
2894                         else
2895                                 M_IMOV_IMM(iptr->val.i, REG_ITMP2);
2896                         M_ITEST(s1);
2897
2898                         switch (iptr->opc) {
2899                         case ICMD_IFEQ_ICONST:
2900                                 M_CMOVEQ(REG_ITMP2, d);
2901                                 break;
2902                         case ICMD_IFNE_ICONST:
2903                                 M_CMOVNE(REG_ITMP2, d);
2904                                 break;
2905                         case ICMD_IFLT_ICONST:
2906                                 M_CMOVLT(REG_ITMP2, d);
2907                                 break;
2908                         case ICMD_IFGE_ICONST:
2909                                 M_CMOVGE(REG_ITMP2, d);
2910                                 break;
2911                         case ICMD_IFGT_ICONST:
2912                                 M_CMOVGT(REG_ITMP2, d);
2913                                 break;
2914                         case ICMD_IFLE_ICONST:
2915                                 M_CMOVLE(REG_ITMP2, d);
2916                                 break;
2917                         }
2918
2919                         store_reg_to_var_int(iptr->dst, d);
2920                         break;
2921
2922
2923                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2924                 case ICMD_LRETURN:
2925
2926                         var_to_reg_int(s1, src, REG_RESULT);
2927                         M_INTMOVE(s1, REG_RESULT);
2928                         goto nowperformreturn;
2929
2930                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2931
2932                         var_to_reg_int(s1, src, REG_RESULT);
2933                         M_INTMOVE(s1, REG_RESULT);
2934
2935 #ifdef ENABLE_VERIFIER
2936                         if (iptr->val.a) {
2937                                 PROFILE_CYCLE_STOP;
2938
2939                                 codegen_addpatchref(cd, cd->mcodeptr,
2940                                                                         PATCHER_athrow_areturn,
2941                                                                         (unresolved_class *) iptr->val.a, 0);
2942
2943                                 if (opt_showdisassemble) {
2944                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2945                                 }
2946
2947                                 PROFILE_CYCLE_START;
2948                         }
2949 #endif /* ENABLE_VERIFIER */
2950                         goto nowperformreturn;
2951
2952                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2953                 case ICMD_DRETURN:
2954
2955                         var_to_reg_flt(s1, src, REG_FRESULT);
2956                         M_FLTMOVE(s1, REG_FRESULT);
2957                         goto nowperformreturn;
2958
2959                 case ICMD_RETURN:      /* ...  ==> ...                                */
2960
2961 nowperformreturn:
2962                         {
2963                         s4 i, p;
2964
2965                         p = parentargs_base;
2966
2967                         /* call trace function */
2968                         if (runverbose) {
2969                                 x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
2970
2971                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2972                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2973
2974                                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
2975                                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2976                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2977                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2978
2979                                 x86_64_mov_imm_reg(cd, (u8) builtin_displaymethodstop, REG_ITMP1);
2980                                 x86_64_call_reg(cd, REG_ITMP1);
2981
2982                                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2983                                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2984
2985                                 x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
2986                         }
2987
2988 #if defined(USE_THREADS)
2989                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2990                                 M_ALD(rd->argintregs[0], REG_SP, rd->memuse * 8);
2991
2992                                 /* we need to save the proper return value */
2993                                 switch (iptr->opc) {
2994                                 case ICMD_IRETURN:
2995                                 case ICMD_ARETURN:
2996                                 case ICMD_LRETURN:
2997                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2998                                         break;
2999                                 case ICMD_FRETURN:
3000                                 case ICMD_DRETURN:
3001                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
3002                                         break;
3003                                 }
3004
3005                                 M_MOV_IMM((ptrint) builtin_monitorexit, REG_ITMP1);
3006                                 M_CALL(REG_ITMP1);
3007
3008                                 /* and now restore the proper return value */
3009                                 switch (iptr->opc) {
3010                                 case ICMD_IRETURN:
3011                                 case ICMD_ARETURN:
3012                                 case ICMD_LRETURN:
3013                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
3014                                         break;
3015                                 case ICMD_FRETURN:
3016                                 case ICMD_DRETURN:
3017                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
3018                                         break;
3019                                 }
3020                         }
3021 #endif
3022
3023                         /* restore saved registers */
3024
3025                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
3026                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
3027                         }
3028                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
3029                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
3030                         }
3031
3032                         /* deallocate stack */
3033
3034                         if (parentargs_base)
3035                                 M_AADD_IMM(parentargs_base * 8, REG_SP);
3036
3037                         /* generate method profiling code */
3038
3039                         PROFILE_CYCLE_STOP;
3040
3041                         M_RET;
3042                         }
3043                         break;
3044
3045
3046                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
3047                         {
3048                                 s4 i, l, *s4ptr;
3049                                 void **tptr;
3050
3051                                 tptr = (void **) iptr->target;
3052
3053                                 s4ptr = iptr->val.a;
3054                                 l = s4ptr[1];                          /* low     */
3055                                 i = s4ptr[2];                          /* high    */
3056
3057                                 var_to_reg_int(s1, src, REG_ITMP1);
3058                                 M_INTMOVE(s1, REG_ITMP1);
3059                                 if (l != 0) {
3060                                         x86_64_alul_imm_reg(cd, X86_64_SUB, l, REG_ITMP1);
3061                                 }
3062                                 i = i - l + 1;
3063
3064                 /* range check */
3065                                 x86_64_alul_imm_reg(cd, X86_64_CMP, i - 1, REG_ITMP1);
3066                                 x86_64_jcc(cd, X86_64_CC_A, 0);
3067
3068                 /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[0]), cd->mcodeptr); */
3069                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3070
3071                                 /* build jump table top down and use address of lowest entry */
3072
3073                 /* s4ptr += 3 + i; */
3074                                 tptr += i;
3075
3076                                 while (--i >= 0) {
3077                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
3078                                         --tptr;
3079                                 }
3080
3081                                 /* length of dataseg after last dseg_addtarget is used by load */
3082
3083                                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2);
3084                                 dseg_adddata(cd, cd->mcodeptr);
3085                                 x86_64_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
3086                                 x86_64_jmp_reg(cd, REG_ITMP1);
3087                         }
3088                         break;
3089
3090
3091                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
3092                         {
3093                                 s4 i, l, val, *s4ptr;
3094                                 void **tptr;
3095
3096                                 tptr = (void **) iptr->target;
3097
3098                                 s4ptr = iptr->val.a;
3099                                 l = s4ptr[0];                          /* default  */
3100                                 i = s4ptr[1];                          /* count    */
3101
3102                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
3103                                 var_to_reg_int(s1, src, REG_ITMP1);    /* reg compare should always be faster */
3104                                 while (--i >= 0) {
3105                                         s4ptr += 2;
3106                                         ++tptr;
3107
3108                                         val = s4ptr[0];
3109                                         x86_64_alul_imm_reg(cd, X86_64_CMP, val, s1);
3110                                         x86_64_jcc(cd, X86_64_CC_E, 0);
3111                                         codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3112                                 }
3113
3114                                 x86_64_jmp_imm(cd, 0);
3115
3116                                 tptr = (void **) iptr->target;
3117                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3118                         }
3119                         break;
3120
3121
3122                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
3123                                         /* op1 = arg count val.a = builtintable entry */
3124
3125                         bte = iptr->val.a;
3126                         md = bte->md;
3127                         goto gen_method;
3128
3129                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
3130                                         /* op1 = arg count, val.a = method pointer    */
3131
3132                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
3133                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
3134                 case ICMD_INVOKEINTERFACE:
3135
3136                         lm = iptr->val.a;
3137
3138                         if (lm == NULL) {
3139                                 unresolved_method *um = iptr->target;
3140                                 md = um->methodref->parseddesc.md;
3141                         } else {
3142                                 md = lm->parseddesc;
3143                         }
3144
3145 gen_method:
3146                         s3 = md->paramcount;
3147
3148                         MCODECHECK((20 * s3) + 128);
3149
3150                         /* copy arguments to registers or stack location */
3151
3152                         for (s3 = s3 - 1; s3 >= 0; s3--, src = src->prev) {
3153                                 if (src->varkind == ARGVAR)
3154                                         continue;
3155                                 if (IS_INT_LNG_TYPE(src->type)) {
3156                                         if (!md->params[s3].inmemory) {
3157                                                 s1 = rd->argintregs[md->params[s3].regoff];
3158                                                 var_to_reg_int(d, src, s1);
3159                                                 M_INTMOVE(d, s1);
3160                                         } else {
3161                                                 var_to_reg_int(d, src, REG_ITMP1);
3162                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
3163                                         }
3164
3165                                 } else {
3166                                         if (!md->params[s3].inmemory) {
3167                                                 s1 = rd->argfltregs[md->params[s3].regoff];
3168                                                 var_to_reg_flt(d, src, s1);
3169                                                 M_FLTMOVE(d, s1);
3170                                         } else {
3171                                                 var_to_reg_flt(d, src, REG_FTMP1);
3172                                                 M_DST(d, REG_SP, md->params[s3].regoff * 8);
3173                                         }
3174                                 }
3175                         }
3176
3177                         /* generate method profiling code */
3178
3179                         PROFILE_CYCLE_STOP;
3180
3181                         switch (iptr->opc) {
3182                         case ICMD_BUILTIN:
3183                                 a = (ptrint) bte->fp;
3184                                 d = md->returntype.type;
3185
3186                                 M_MOV_IMM(a, REG_ITMP1);
3187                                 M_CALL(REG_ITMP1);
3188
3189                                 /* if op1 == true, we need to check for an exception */
3190
3191                                 if (iptr->op1 == true) {
3192                                         M_TEST(REG_RESULT);
3193                                         M_BEQ(0);
3194                                         codegen_addxexceptionrefs(cd, cd->mcodeptr);
3195                                 }
3196                                 break;
3197
3198                         case ICMD_INVOKESPECIAL:
3199                                 M_TEST(rd->argintregs[0]);
3200                                 M_BEQ(0);
3201                                 codegen_addxnullrefs(cd, cd->mcodeptr);
3202
3203                                 /* first argument contains pointer */
3204 /*                              gen_nullptr_check(rd->argintregs[0]); */
3205
3206                                 /* access memory for hardware nullptr */
3207 /*                              x86_64_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); */
3208
3209                                 /* fall through */
3210
3211                         case ICMD_INVOKESTATIC:
3212                                 if (lm == NULL) {
3213                                         unresolved_method *um = iptr->target;
3214
3215                                         codegen_addpatchref(cd, cd->mcodeptr,
3216                                                                                 PATCHER_invokestatic_special, um, 0);
3217
3218                                         if (opt_showdisassemble) {
3219                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3220                                         }
3221
3222                                         a = 0;
3223                                         d = um->methodref->parseddesc.md->returntype.type;
3224
3225                                 } else {
3226                                         a = (ptrint) lm->stubroutine;
3227                                         d = lm->parseddesc->returntype.type;
3228                                 }
3229
3230                                 M_MOV_IMM(a, REG_ITMP2);
3231                                 M_CALL(REG_ITMP2);
3232                                 break;
3233
3234                         case ICMD_INVOKEVIRTUAL:
3235                                 gen_nullptr_check(rd->argintregs[0]);
3236
3237                                 if (lm == NULL) {
3238                                         unresolved_method *um = iptr->target;
3239
3240                                         codegen_addpatchref(cd, cd->mcodeptr,
3241                                                                                 PATCHER_invokevirtual, um, 0);
3242
3243                                         if (opt_showdisassemble) {
3244                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3245                                         }
3246
3247                                         s1 = 0;
3248                                         d = um->methodref->parseddesc.md->returntype.type;
3249
3250                                 } else {
3251                                         s1 = OFFSET(vftbl_t, table[0]) +
3252                                                 sizeof(methodptr) * lm->vftblindex;
3253                                         d = lm->parseddesc->returntype.type;
3254                                 }
3255
3256                                 x86_64_mov_membase_reg(cd, rd->argintregs[0],
3257                                                                            OFFSET(java_objectheader, vftbl),
3258                                                                            REG_ITMP2);
3259                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP1);
3260                                 M_CALL(REG_ITMP1);
3261                                 break;
3262
3263                         case ICMD_INVOKEINTERFACE:
3264                                 gen_nullptr_check(rd->argintregs[0]);
3265
3266                                 if (lm == NULL) {
3267                                         unresolved_method *um = iptr->target;
3268
3269                                         codegen_addpatchref(cd, cd->mcodeptr,
3270                                                                                 PATCHER_invokeinterface, um, 0);
3271
3272                                         if (opt_showdisassemble) {
3273                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3274                                         }
3275
3276                                         s1 = 0;
3277                                         s2 = 0;
3278                                         d = um->methodref->parseddesc.md->returntype.type;
3279
3280                                 } else {
3281                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3282                                                 sizeof(methodptr) * lm->class->index;
3283
3284                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3285
3286                                         d = lm->parseddesc->returntype.type;
3287                                 }
3288
3289                                 M_ALD(REG_ITMP2, rd->argintregs[0],
3290                                           OFFSET(java_objectheader, vftbl));
3291                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP2);
3292                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s2, REG_ITMP1);
3293                                 M_CALL(REG_ITMP1);
3294                                 break;
3295                         }
3296
3297                         /* generate method profiling code */
3298
3299                         PROFILE_CYCLE_START;
3300
3301                         /* d contains return type */
3302
3303                         if (d != TYPE_VOID) {
3304                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3305                                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3306                                         M_INTMOVE(REG_RESULT, s1);
3307                                         store_reg_to_var_int(iptr->dst, s1);
3308                                 } else {
3309                                         s1 = reg_of_var(rd, iptr->dst, REG_FRESULT);
3310                                         M_FLTMOVE(REG_FRESULT, s1);
3311                                         store_reg_to_var_flt(iptr->dst, s1);
3312                                 }
3313                         }
3314                         break;
3315
3316
3317                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3318
3319                                       /* op1:   0 == array, 1 == class                */
3320                                       /* val.a: (classinfo *) superclass              */
3321
3322                         /*  superclass is an interface:
3323                          *
3324                          *  OK if ((sub == NULL) ||
3325                          *         (sub->vftbl->interfacetablelength > super->index) &&
3326                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3327                          *
3328                          *  superclass is a class:
3329                          *
3330                          *  OK if ((sub == NULL) || (0
3331                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3332                          *         super->vftbl->diffval));
3333                          */
3334
3335                         if (iptr->op1 == 1) {
3336                                 /* object type cast-check */
3337
3338                                 classinfo *super;
3339                                 vftbl_t   *supervftbl;
3340                                 s4         superindex;
3341
3342                                 super = (classinfo *) iptr->val.a;
3343
3344                                 if (!super) {
3345                                         superindex = 0;
3346                                         supervftbl = NULL;
3347
3348                                 } else {
3349                                         superindex = super->index;
3350                                         supervftbl = super->vftbl;
3351                                 }
3352
3353 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3354                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3355 #endif
3356                                 var_to_reg_int(s1, src, REG_ITMP1);
3357
3358                                 /* calculate interface checkcast code size */
3359
3360                                 s2 = 3; /* mov_membase_reg */
3361                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3362
3363                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3364                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3365                                         3 /* test */ + 6 /* jcc */;
3366
3367                                 if (!super)
3368                                         s2 += (opt_showdisassemble ? 5 : 0);
3369
3370                                 /* calculate class checkcast code size */
3371
3372                                 s3 = 3; /* mov_membase_reg */
3373                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3374                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3375
3376 #if 0
3377                                 if (s1 != REG_ITMP1) {
3378                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3379                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3380                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3381                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3382                                         a += 3;    /* sub */
3383
3384                                 } else
3385 #endif
3386                                         {
3387                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3388                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3389                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3390                                         }
3391
3392                                 s3 += 3 /* cmp */ + 6 /* jcc */;
3393
3394                                 if (!super)
3395                                         s3 += (opt_showdisassemble ? 5 : 0);
3396
3397                                 /* if class is not resolved, check which code to call */
3398
3399                                 if (!super) {
3400                                         M_TEST(s1);
3401                                         M_BEQ(6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3402
3403                                         codegen_addpatchref(cd, cd->mcodeptr,
3404                                                                                 PATCHER_checkcast_instanceof_flags,
3405                                                                                 (constant_classref *) iptr->target, 0);
3406
3407                                         if (opt_showdisassemble) {
3408                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3409                                         }
3410
3411                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
3412                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
3413                                         M_BEQ(s2 + 5);
3414                                 }
3415
3416                                 /* interface checkcast code */
3417
3418                                 if (!super || (super->flags & ACC_INTERFACE)) {
3419                                         if (super) {
3420                                                 M_TEST(s1);
3421                                                 M_BEQ(s2);
3422                                         }
3423
3424                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3425
3426                                         if (!super) {
3427                                                 codegen_addpatchref(cd, cd->mcodeptr,
3428                                                                                         PATCHER_checkcast_instanceof_interface,
3429                                                                                         (constant_classref *) iptr->target, 0);
3430
3431                                                 if (opt_showdisassemble) {
3432                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3433                                                 }
3434                                         }
3435
3436                                         x86_64_movl_membase32_reg(cd, REG_ITMP2,
3437                                                                                           OFFSET(vftbl_t, interfacetablelength),
3438                                                                                           REG_ITMP3);
3439                                         /* XXX TWISTI: should this be int arithmetic? */
3440                                         M_LSUB_IMM32(superindex, REG_ITMP3);
3441                                         M_TEST(REG_ITMP3);
3442                                         M_BLE(0);
3443                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3444                                         x86_64_mov_membase32_reg(cd, REG_ITMP2,
3445                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
3446                                                                                          superindex * sizeof(methodptr*),
3447                                                                                          REG_ITMP3);
3448                                         M_TEST(REG_ITMP3);
3449                                         M_BEQ(0);
3450                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3451
3452                                         if (!super)
3453                                                 M_JMP_IMM(s3);
3454                                 }
3455
3456                                 /* class checkcast code */
3457
3458                                 if (!super || !(super->flags & ACC_INTERFACE)) {
3459                                         if (super) {
3460                                                 M_TEST(s1);
3461                                                 M_BEQ(s3);
3462                                         }
3463
3464                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3465
3466                                         if (!super) {
3467                                                 codegen_addpatchref(cd, cd->mcodeptr,
3468                                                                                         PATCHER_checkcast_class,
3469                                                                                         (constant_classref *) iptr->target,
3470                                                                                         0);
3471
3472                                                 if (opt_showdisassemble) {
3473                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3474                                                 }
3475                                         }
3476
3477                                         M_MOV_IMM((ptrint) supervftbl, REG_ITMP3);
3478 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3479                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3480 #endif
3481                                         x86_64_movl_membase32_reg(cd, REG_ITMP2,
3482                                                                                           OFFSET(vftbl_t, baseval),
3483                                                                                           REG_ITMP2);
3484                                         /*                                      if (s1 != REG_ITMP1) { */
3485                                         /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3486                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
3487                                         /*                                                                                              REG_ITMP1); */
3488                                         /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3489                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3490                                         /*                                                                                              REG_ITMP3); */
3491                                         /*  #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
3492                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3493                                         /*  #endif */
3494                                         /*                                              x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP1, REG_ITMP2); */
3495
3496                                         /*                                      } else { */
3497                                         x86_64_movl_membase32_reg(cd, REG_ITMP3,
3498                                                                                           OFFSET(vftbl_t, baseval),
3499                                                                                           REG_ITMP3);
3500                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3501                                         M_MOV_IMM((ptrint) supervftbl, REG_ITMP3);
3502                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3503                                         /*                                      } */
3504 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3505                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3506 #endif
3507                                         M_CMP(REG_ITMP3, REG_ITMP2);
3508                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3509                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3510                                 }
3511                                 d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3512
3513                         } else {
3514                                 /* array type cast-check */
3515
3516                                 var_to_reg_int(s1, src, REG_ITMP1);
3517                                 M_INTMOVE(s1, rd->argintregs[0]);
3518
3519                                 if (iptr->val.a == NULL) {
3520                                         codegen_addpatchref(cd, cd->mcodeptr,
3521                                                                                 PATCHER_builtin_arraycheckcast,
3522                                                                                 (constant_classref *) iptr->target, 0);
3523
3524                                         if (opt_showdisassemble) {
3525                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3526                                         }
3527                                 }
3528
3529                                 M_MOV_IMM((ptrint) iptr->val.a, rd->argintregs[1]);
3530                                 M_MOV_IMM((ptrint) BUILTIN_arraycheckcast, REG_ITMP1);
3531                                 M_CALL(REG_ITMP1);
3532                                 M_TEST(REG_RESULT);
3533                                 M_BEQ(0);
3534                                 codegen_addxcastrefs(cd, cd->mcodeptr);
3535
3536                                 var_to_reg_int(s1, src, REG_ITMP1);
3537                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
3538                         }
3539                         M_INTMOVE(s1, d);
3540                         store_reg_to_var_int(iptr->dst, d);
3541                         break;
3542
3543                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3544
3545                                       /* op1:   0 == array, 1 == class                */
3546                                       /* val.a: (classinfo *) superclass              */
3547
3548                         /*  superclass is an interface:
3549                          *
3550                          *  return (sub != NULL) &&
3551                          *         (sub->vftbl->interfacetablelength > super->index) &&
3552                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3553                          *
3554                          *  superclass is a class:
3555                          *
3556                          *  return ((sub != NULL) && (0
3557                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3558                          *          super->vftbl->diffvall));
3559                          */
3560
3561                         {
3562                         classinfo *super;
3563                         vftbl_t   *supervftbl;
3564                         s4         superindex;
3565
3566                         super = (classinfo *) iptr->val.a;
3567
3568                         if (!super) {
3569                                 superindex = 0;
3570                                 supervftbl = NULL;
3571
3572                         } else {
3573                                 superindex = super->index;
3574                                 supervftbl = super->vftbl;
3575                         }
3576
3577 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3578             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3579 #endif
3580
3581                         var_to_reg_int(s1, src, REG_ITMP1);
3582                         d = reg_of_var(rd, iptr->dst, REG_ITMP2);
3583                         if (s1 == d) {
3584                                 M_INTMOVE(s1, REG_ITMP1);
3585                                 s1 = REG_ITMP1;
3586                         }
3587
3588                         /* calculate interface instanceof code size */
3589
3590                         s2 = 3; /* mov_membase_reg */
3591                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3592                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3593                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3594                                 3 /* test */ + 4 /* setcc */;
3595
3596                         if (!super)
3597                                 s2 += (opt_showdisassemble ? 5 : 0);
3598
3599                         /* calculate class instanceof code size */
3600
3601                         s3 = 3; /* mov_membase_reg */
3602                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3603                         s3 += 10; /* mov_imm_reg */
3604                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3605                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3606                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3607                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3608                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3609                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3610                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3611
3612                         if (!super)
3613                                 s3 += (opt_showdisassemble ? 5 : 0);
3614
3615                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3616
3617                         /* if class is not resolved, check which code to call */
3618
3619                         if (!super) {
3620                                 x86_64_test_reg_reg(cd, s1, s1);
3621                                 x86_64_jcc(cd, X86_64_CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3622                                                                                          7 + 6 + s2 + 5 + s3));
3623
3624                                 codegen_addpatchref(cd, cd->mcodeptr,
3625                                                                         PATCHER_checkcast_instanceof_flags,
3626                                                                         (constant_classref *) iptr->target, 0);
3627
3628                                 if (opt_showdisassemble) {
3629                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3630                                 }
3631
3632                                 x86_64_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3633                                 x86_64_alul_imm_reg(cd, X86_64_AND, ACC_INTERFACE, REG_ITMP3);
3634                                 x86_64_jcc(cd, X86_64_CC_Z, s2 + 5);
3635                         }
3636
3637                         /* interface instanceof code */
3638
3639                         if (!super || (super->flags & ACC_INTERFACE)) {
3640                                 if (super) {
3641                                         x86_64_test_reg_reg(cd, s1, s1);
3642                                         x86_64_jcc(cd, X86_64_CC_Z, s2);
3643                                 }
3644
3645                                 x86_64_mov_membase_reg(cd, s1,
3646                                                                            OFFSET(java_objectheader, vftbl),
3647                                                                            REG_ITMP1);
3648                                 if (!super) {
3649                                         codegen_addpatchref(cd, cd->mcodeptr,
3650                                                                                 PATCHER_checkcast_instanceof_interface,
3651                                                                                 (constant_classref *) iptr->target, 0);
3652
3653                                         if (opt_showdisassemble) {
3654                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3655                                         }
3656                                 }
3657
3658                                 x86_64_movl_membase32_reg(cd, REG_ITMP1,
3659                                                                                   OFFSET(vftbl_t, interfacetablelength),
3660                                                                                   REG_ITMP3);
3661                                 x86_64_alu_imm32_reg(cd, X86_64_SUB, superindex, REG_ITMP3);
3662                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3663
3664                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3665
3666                                 x86_64_jcc(cd, X86_64_CC_LE, a);
3667                                 x86_64_mov_membase32_reg(cd, REG_ITMP1,
3668                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3669                                                                                  superindex * sizeof(methodptr*),
3670                                                                                  REG_ITMP1);
3671                                 x86_64_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3672                                 x86_64_setcc_reg(cd, X86_64_CC_NE, d);
3673
3674                                 if (!super)
3675                                         x86_64_jmp_imm(cd, s3);
3676                         }
3677
3678                         /* class instanceof code */
3679
3680                         if (!super || !(super->flags & ACC_INTERFACE)) {
3681                                 if (super) {
3682                                         x86_64_test_reg_reg(cd, s1, s1);
3683                                         x86_64_jcc(cd, X86_64_CC_E, s3);
3684                                 }
3685
3686                                 x86_64_mov_membase_reg(cd, s1,
3687                                                                            OFFSET(java_objectheader, vftbl),
3688                                                                            REG_ITMP1);
3689
3690                                 if (!super) {
3691                                         codegen_addpatchref(cd, cd->mcodeptr,
3692                                                                                 PATCHER_instanceof_class,
3693                                                                                 (constant_classref *) iptr->target, 0);
3694
3695                                         if (opt_showdisassemble) {
3696                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3697                                         }
3698                                 }
3699
3700                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3701 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3702                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3703 #endif
3704                                 x86_64_movl_membase_reg(cd, REG_ITMP1,
3705                                                                                 OFFSET(vftbl_t, baseval),
3706                                                                                 REG_ITMP1);
3707                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3708                                                                                 OFFSET(vftbl_t, diffval),
3709                                                                                 REG_ITMP3);
3710                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3711                                                                                 OFFSET(vftbl_t, baseval),
3712                                                                                 REG_ITMP2);
3713 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3714                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3715 #endif
3716                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
3717                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d); /* may be REG_ITMP2 */
3718                                 x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP3, REG_ITMP1);
3719                                 x86_64_setcc_reg(cd, X86_64_CC_BE, d);
3720                         }
3721                         store_reg_to_var_int(iptr->dst, d);
3722                         }
3723                         break;
3724
3725                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3726                                          /* op1 = dimension, val.a = class            */
3727
3728                         /* check for negative sizes and copy sizes to stack if necessary  */
3729
3730                         MCODECHECK((10 * 4 * iptr->op1) + 5 + 10 * 8);
3731
3732                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3733                                 /* copy SAVEDVAR sizes to stack */
3734
3735                                 if (src->varkind != ARGVAR) {
3736                                         var_to_reg_int(s2, src, REG_ITMP1);
3737                                         M_LST(s2, REG_SP, s1 * 8);
3738                                 }
3739                         }
3740
3741                         /* is a patcher function set? */
3742
3743                         if (iptr->val.a == NULL) {
3744                                 codegen_addpatchref(cd, cd->mcodeptr,
3745                                                                         PATCHER_builtin_multianewarray,
3746                                                                         (constant_classref *) iptr->target, 0);
3747
3748                                 if (opt_showdisassemble) {
3749                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3750                                 }
3751
3752                                 a = 0;
3753
3754                         } else {
3755                                 a = (ptrint) iptr->val.a;
3756                         }
3757
3758                         /* a0 = dimension count */
3759
3760                         M_MOV_IMM(iptr->op1, rd->argintregs[0]);
3761
3762                         /* a1 = arrayvftbl */
3763
3764                         M_MOV_IMM((ptrint) iptr->val.a, rd->argintregs[1]);
3765
3766                         /* a2 = pointer to dimensions = stack pointer */
3767
3768                         M_MOV(REG_SP, rd->argintregs[2]);
3769
3770                         M_MOV_IMM((ptrint) BUILTIN_multianewarray, REG_ITMP1);
3771                         M_CALL(REG_ITMP1);
3772
3773                         /* check for exception before result assignment */
3774
3775                         M_TEST(REG_RESULT);
3776                         M_BEQ(0);
3777                         codegen_addxexceptionrefs(cd, cd->mcodeptr);
3778
3779                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3780                         M_INTMOVE(REG_RESULT, s1);
3781                         store_reg_to_var_int(iptr->dst, s1);
3782                         break;
3783
3784                 default:
3785                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3786                         return false;
3787         } /* switch */
3788
3789         } /* for instruction */
3790
3791         /* copy values to interface registers */
3792
3793         src = bptr->outstack;
3794         len = bptr->outdepth;
3795         MCODECHECK(512);
3796 #if defined(ENABLE_LSRA)
3797         if (!opt_lsra)
3798 #endif
3799         while (src) {
3800                 len--;
3801                 if ((src->varkind != STACKVAR)) {
3802                         s2 = src->type;
3803                         if (IS_FLT_DBL_TYPE(s2)) {
3804                                 var_to_reg_flt(s1, src, REG_FTMP1);
3805                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3806                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3807
3808                                 } else {
3809                                         x86_64_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3810                                 }
3811
3812                         } else {
3813                                 var_to_reg_int(s1, src, REG_ITMP1);
3814                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3815                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3816
3817                                 } else {
3818                                         x86_64_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3819                                 }
3820                         }
3821                 }
3822                 src = src->prev;
3823         }
3824
3825         /* At the end of a basic block we may have to append some nops,
3826            because the patcher stub calling code might be longer than the
3827            actual instruction. So codepatching does not change the
3828            following block unintentionally. */
3829
3830         if (cd->mcodeptr < cd->lastmcodeptr) {
3831                 while (cd->mcodeptr < cd->lastmcodeptr) {
3832                         M_NOP;
3833                 }
3834         }
3835
3836         } /* if (bptr -> flags >= BBREACHED) */
3837         } /* for basic block */
3838
3839         dseg_createlinenumbertable(cd);
3840
3841         {
3842
3843         u1        *xcodeptr;
3844         branchref *bref;
3845
3846         /* generate ArithmeticException stubs */
3847
3848         xcodeptr = NULL;
3849
3850         for (bref = cd->xdivrefs; bref != NULL; bref = bref->next) {
3851                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3852                                   bref->branchpos,
3853                                                   cd->mcodeptr - cd->mcodebase);
3854
3855                 MCODECHECK(512);
3856
3857                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3858                 dseg_adddata(cd, cd->mcodeptr);
3859                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3860
3861                 if (xcodeptr != NULL) {
3862                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3863
3864                 } else {
3865                         xcodeptr = cd->mcodeptr;
3866
3867                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3868                         M_MOV(REG_SP, rd->argintregs[1]);
3869                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3870                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3871
3872                         M_ASUB_IMM(2 * 8, REG_SP);
3873                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3874
3875                         M_MOV_IMM((ptrint) stacktrace_inline_arithmeticexception,
3876                                           REG_ITMP3);
3877                         M_CALL(REG_ITMP3);
3878
3879                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3880                         M_AADD_IMM(2 * 8, REG_SP);
3881
3882                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3883                         M_JMP(REG_ITMP3);
3884                 }
3885         }
3886
3887         /* generate ArrayIndexOutOfBoundsException stubs */
3888
3889         xcodeptr = NULL;
3890
3891         for (bref = cd->xboundrefs; bref != NULL; bref = bref->next) {
3892                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3893                                   bref->branchpos,
3894                                                   cd->mcodeptr - cd->mcodebase);
3895
3896                 MCODECHECK(512);
3897
3898                 /* move index register into REG_ITMP1 */
3899
3900                 M_MOV(bref->reg, REG_ITMP1);                              /* 3 bytes  */
3901
3902                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3903                 dseg_adddata(cd, cd->mcodeptr);
3904                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3905
3906                 if (xcodeptr != NULL) {
3907                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3908
3909                 } else {
3910                         xcodeptr = cd->mcodeptr;
3911
3912                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3913                         M_MOV(REG_SP, rd->argintregs[1]);
3914                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3915                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3916                         M_MOV(REG_ITMP1, rd->argintregs[4]);
3917
3918                         M_ASUB_IMM(2 * 8, REG_SP);
3919                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3920
3921                         M_MOV_IMM((ptrint) stacktrace_inline_arrayindexoutofboundsexception,
3922                                           REG_ITMP3);
3923                         M_CALL(REG_ITMP3);
3924
3925                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3926                         M_AADD_IMM(2 * 8, REG_SP);
3927
3928                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3929                         M_JMP(REG_ITMP3);
3930                 }
3931         }
3932
3933         /* generate ArrayStoreException stubs */
3934
3935         xcodeptr = NULL;
3936
3937         for (bref = cd->xstorerefs; bref != NULL; bref = bref->next) {
3938                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3939                                   bref->branchpos,
3940                                                   cd->mcodeptr - cd->mcodebase);
3941
3942                 MCODECHECK(512);
3943
3944                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3945                 dseg_adddata(cd, cd->mcodeptr);
3946                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3947
3948                 if (xcodeptr != NULL) {
3949                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3950
3951                 } else {
3952                         xcodeptr = cd->mcodeptr;
3953
3954                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3955                         M_MOV(REG_SP, rd->argintregs[1]);
3956                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3957                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3958
3959                         M_ASUB_IMM(2 * 8, REG_SP);
3960                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3961
3962                         M_MOV_IMM((ptrint) stacktrace_inline_arraystoreexception,
3963                                           REG_ITMP3);
3964                         M_CALL(REG_ITMP3);
3965
3966                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3967                         M_AADD_IMM(2 * 8, REG_SP);
3968
3969                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3970                         M_JMP(REG_ITMP3);
3971                 }
3972         }
3973
3974         /* generate ClassCastException stubs */
3975
3976         xcodeptr = NULL;
3977
3978         for (bref = cd->xcastrefs; bref != NULL; bref = bref->next) {
3979                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3980                                   bref->branchpos,
3981                                                   cd->mcodeptr - cd->mcodebase);
3982
3983                 MCODECHECK(512);
3984
3985                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3986                 dseg_adddata(cd, cd->mcodeptr);
3987                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3988
3989                 if (xcodeptr != NULL) {
3990                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3991
3992                 } else {
3993                         xcodeptr = cd->mcodeptr;
3994
3995                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3996                         M_MOV(REG_SP, rd->argintregs[1]);
3997                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3998                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3999
4000                         M_ASUB_IMM(2 * 8, REG_SP);
4001                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
4002
4003                         M_MOV_IMM((ptrint) stacktrace_inline_classcastexception, REG_ITMP3);
4004                         M_CALL(REG_ITMP3);
4005
4006                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
4007                         M_AADD_IMM(2 * 8, REG_SP);
4008
4009                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
4010                         M_JMP(REG_ITMP3);
4011                 }
4012         }
4013
4014         /* generate NullpointerException stubs */
4015
4016         xcodeptr = NULL;
4017
4018         for (bref = cd->xnullrefs; bref != NULL; bref = bref->next) {
4019                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
4020                                   bref->branchpos,
4021                                                   cd->mcodeptr - cd->mcodebase);
4022
4023                 MCODECHECK(512);
4024
4025                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
4026                 dseg_adddata(cd, cd->mcodeptr);
4027                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
4028
4029                 if (xcodeptr != NULL) {
4030                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
4031
4032                 } else {
4033                         xcodeptr = cd->mcodeptr;
4034
4035                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
4036                         M_MOV(REG_SP, rd->argintregs[1]);
4037                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
4038                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
4039
4040                         M_ASUB_IMM(2 * 8, REG_SP);
4041                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
4042
4043                         M_MOV_IMM((ptrint) stacktrace_inline_nullpointerexception,
4044                                           REG_ITMP3);
4045                         M_CALL(REG_ITMP3);
4046
4047                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
4048                         M_AADD_IMM(2 * 8, REG_SP);
4049
4050                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
4051                         M_JMP(REG_ITMP3);
4052                 }
4053         }
4054
4055         /* generate exception check stubs */
4056
4057         xcodeptr = NULL;
4058
4059         for (bref = cd->xexceptionrefs; bref != NULL; bref = bref->next) {
4060                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
4061                                   bref->branchpos,
4062                                                   cd->mcodeptr - cd->mcodebase);
4063
4064                 MCODECHECK(512);
4065
4066                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
4067                 dseg_adddata(cd, cd->mcodeptr);
4068                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
4069
4070                 if (xcodeptr != NULL) {
4071                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
4072
4073                 } else {
4074                         xcodeptr = cd->mcodeptr;
4075
4076                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
4077                         M_MOV(REG_SP, rd->argintregs[1]);
4078                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
4079                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
4080
4081                         M_ASUB_IMM(2 * 8, REG_SP);
4082                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
4083
4084                         M_MOV_IMM((ptrint) stacktrace_inline_fillInStackTrace, REG_ITMP3);
4085                         M_CALL(REG_ITMP3);
4086
4087                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
4088                         M_AADD_IMM(2 * 8, REG_SP);
4089
4090                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
4091                         M_JMP(REG_ITMP3);
4092                 }
4093         }
4094
4095         /* generate code patching stub call code */
4096
4097         {
4098                 patchref    *pref;
4099                 codegendata *tmpcd;
4100                 ptrint       mcode;
4101
4102                 tmpcd = DNEW(codegendata);
4103
4104                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4105                         /* check size of code segment */
4106
4107                         MCODECHECK(512);
4108
4109                         /* Get machine code which is patched back in later. A             */
4110                         /* `call rel32' is 5 bytes long (but read 8 bytes).               */
4111
4112                         xcodeptr = cd->mcodebase + pref->branchpos;
4113                         mcode = *((ptrint *) xcodeptr);
4114
4115                         /* patch in `call rel32' to call the following code               */
4116
4117                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
4118                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
4119
4120                         /* move pointer to java_objectheader onto stack */
4121
4122 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4123                         /* create a virtual java_objectheader */
4124
4125                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4126                         a = dseg_addaddress(cd, NULL);                      /* vftbl      */
4127
4128                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP3);
4129                         M_PUSH(REG_ITMP3);
4130 #else
4131                         M_PUSH_IMM(0);
4132 #endif
4133
4134                         /* move machine code bytes and classinfo pointer into registers */
4135
4136                         M_MOV_IMM((ptrint) mcode, REG_ITMP3);
4137                         M_PUSH(REG_ITMP3);
4138                         M_MOV_IMM((ptrint) pref->ref, REG_ITMP3);
4139                         M_PUSH(REG_ITMP3);
4140                         M_MOV_IMM((ptrint) pref->disp, REG_ITMP3);
4141                         M_PUSH(REG_ITMP3);
4142
4143                         M_MOV_IMM((ptrint) pref->patcher, REG_ITMP3);
4144                         M_PUSH(REG_ITMP3);
4145
4146                         M_MOV_IMM((ptrint) asm_wrapper_patcher, REG_ITMP3);
4147                         M_JMP(REG_ITMP3);
4148                 }
4149         }
4150         }
4151
4152         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
4153
4154         /* everything's ok */
4155
4156         return true;
4157 }
4158
4159
4160 /* createcompilerstub **********************************************************
4161
4162    Creates a stub routine which calls the compiler.
4163
4164 *******************************************************************************/
4165
4166 #define COMPILERSTUB_DATASIZE    2 * SIZEOF_VOID_P
4167 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
4168
4169 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
4170
4171
4172 u1 *createcompilerstub(methodinfo *m)
4173 {
4174         u1          *s;                     /* memory to hold the stub            */
4175         ptrint      *d;
4176         codegendata *cd;
4177         s4           dumpsize;
4178
4179         s = CNEW(u1, COMPILERSTUB_SIZE);
4180
4181         /* set data pointer and code pointer */
4182
4183         d = (ptrint *) s;
4184         s = s + COMPILERSTUB_DATASIZE;
4185
4186         /* mark start of dump memory area */
4187
4188         dumpsize = dump_size();
4189
4190         cd = DNEW(codegendata);
4191         cd->mcodeptr = s;
4192
4193         /* Store the methodinfo* in the same place as in the methodheader
4194            for compiled methods. */
4195
4196         d[0] = (ptrint) asm_call_jit_compiler;
4197         d[1] = (ptrint) m;
4198
4199         /* code for the stub */
4200
4201         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 1 * SIZEOF_VOID_P));       /* methodinfo  */
4202         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 2 * SIZEOF_VOID_P));  /* compiler pointer */
4203         M_JMP(REG_ITMP3);
4204
4205 #if defined(ENABLE_STATISTICS)
4206         if (opt_stat)
4207                 count_cstub_len += COMPILERSTUB_SIZE;
4208 #endif
4209
4210         /* release dump area */
4211
4212         dump_release(dumpsize);
4213
4214         return s;
4215 }
4216
4217
4218 /* createnativestub ************************************************************
4219
4220    Creates a stub routine which calls a native method.
4221
4222 *******************************************************************************/
4223
4224 u1 *createnativestub(functionptr f, methodinfo *m, codegendata *cd,
4225                                          registerdata *rd, methoddesc *nmd)
4226 {
4227         methoddesc *md;
4228         s4          stackframesize;         /* size of stackframe if needed       */
4229         s4          nativeparams;
4230         s4          i, j;                   /* count variables                    */
4231         s4          t;
4232         s4          s1, s2;
4233
4234         /* initialize variables */
4235
4236         md = m->parseddesc;
4237         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
4238
4239         /* calculate stack frame size */
4240
4241         stackframesize =
4242                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
4243                 sizeof(localref_table) / SIZEOF_VOID_P +
4244                 INT_ARG_CNT + FLT_ARG_CNT + 1 +         /* + 1 for function address   */
4245                 nmd->memuse;
4246
4247         if (!(stackframesize & 0x1))                /* keep stack 16-byte aligned */
4248                 stackframesize++;
4249
4250         /* create method header */
4251
4252         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
4253         (void) dseg_adds4(cd, stackframesize * 8);              /* FrameSize      */
4254         (void) dseg_adds4(cd, 0);                               /* IsSync         */
4255         (void) dseg_adds4(cd, 0);                               /* IsLeaf         */
4256         (void) dseg_adds4(cd, 0);                               /* IntSave        */
4257         (void) dseg_adds4(cd, 0);                               /* FltSave        */
4258         (void) dseg_addlinenumbertablesize(cd);
4259         (void) dseg_adds4(cd, 0);                               /* ExTableSize    */
4260
4261         /* initialize mcode variables */
4262
4263         cd->mcodeptr = (u1 *) cd->mcodebase;
4264         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
4265
4266         /* generate native method profiling code */
4267
4268         if (opt_prof) {
4269                 /* count frequency */
4270
4271                 M_MOV_IMM((ptrint) m, REG_ITMP2);
4272                 M_IINC_MEMBASE(REG_ITMP2, OFFSET(methodinfo, frequency));
4273         }
4274
4275         /* generate stub code */
4276
4277         M_ASUB_IMM(stackframesize * 8, REG_SP);
4278
4279         if (runverbose) {
4280                 /* save integer and float argument registers */
4281
4282                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4283                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4284                                 M_LST(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4285
4286                 for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4287                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4288                                 M_DST(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4289
4290                 /* show integer hex code for float arguments */
4291
4292                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++) {
4293                         /* if the paramtype is a float, we have to right shift all
4294                            following integer registers */
4295
4296                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type)) {
4297                                 for (s1 = INT_ARG_CNT - 2; s1 >= i; s1--)
4298                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
4299
4300                                 x86_64_movd_freg_reg(cd, rd->argfltregs[j], rd->argintregs[i]);
4301                                 j++;
4302                         }
4303                 }
4304
4305                 M_MOV_IMM((ptrint) m, REG_ITMP1);
4306                 M_AST(REG_ITMP1, REG_SP, 0 * 8);
4307                 M_MOV_IMM((ptrint) builtin_trace_args, REG_ITMP1);
4308                 M_CALL(REG_ITMP1);
4309
4310                 /* restore integer and float argument registers */
4311
4312                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4313                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4314                                 M_LLD(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4315
4316                 for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4317                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4318                                 M_DLD(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4319         }
4320
4321
4322         /* get function address (this must happen before the stackframeinfo) */
4323
4324 #if !defined(ENABLE_STATICVM)
4325         if (f == NULL) {
4326                 codegen_addpatchref(cd, cd->mcodeptr, PATCHER_resolve_native, m, 0);
4327
4328                 if (opt_showdisassemble) {
4329                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
4330                 }
4331         }
4332 #endif
4333
4334         M_MOV_IMM((ptrint) f, REG_ITMP3);
4335
4336
4337         /* save integer and float argument registers */
4338
4339         for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4340                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4341                         M_LST(rd->argintregs[j++], REG_SP, i * 8);
4342
4343         for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4344                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4345                         M_DST(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4346
4347         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4348
4349         /* create dynamic stack info */
4350
4351         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4352         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[1]);
4353         M_ALEA(REG_SP, stackframesize * 8 + SIZEOF_VOID_P, rd->argintregs[2]);
4354         M_ALD(rd->argintregs[3], REG_SP, stackframesize * 8);
4355         M_MOV_IMM((ptrint) codegen_start_native_call, REG_ITMP1);
4356         M_CALL(REG_ITMP1);
4357
4358         /* restore integer and float argument registers */
4359
4360         for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4361                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4362                         M_LLD(rd->argintregs[j++], REG_SP, i * 8);
4363
4364         for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4365                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4366                         M_DLD(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4367
4368         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4369
4370
4371         /* copy or spill arguments to new locations */
4372
4373         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
4374                 t = md->paramtypes[i].type;
4375
4376                 if (IS_INT_LNG_TYPE(t)) {
4377                         if (!md->params[i].inmemory) {
4378                                 s1 = rd->argintregs[md->params[i].regoff];
4379
4380                                 if (!nmd->params[j].inmemory) {
4381                                         s2 = rd->argintregs[nmd->params[j].regoff];
4382                                         M_INTMOVE(s1, s2);
4383
4384                                 } else {
4385                                         s2 = nmd->params[j].regoff;
4386                                         M_LST(s1, REG_SP, s2 * 8);
4387                                 }
4388
4389                         } else {
4390                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4391                                 s2 = nmd->params[j].regoff;
4392                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
4393                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
4394                         }
4395
4396                 } else {
4397                         /* We only copy spilled float arguments, as the float argument    */
4398                         /* registers keep unchanged.                                      */
4399
4400                         if (md->params[i].inmemory) {
4401                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4402                                 s2 = nmd->params[j].regoff;
4403                                 M_DLD(REG_FTMP1, REG_SP, s1 * 8);
4404                                 M_DST(REG_FTMP1, REG_SP, s2 * 8);
4405                         }
4406                 }
4407         }
4408
4409         /* put class into second argument register */
4410
4411         if (m->flags & ACC_STATIC)
4412                 M_MOV_IMM((ptrint) m->class, rd->argintregs[1]);
4413
4414         /* put env into first argument register */
4415
4416         M_MOV_IMM((ptrint) &env, rd->argintregs[0]);
4417
4418         /* do the native function call */
4419
4420         M_CALL(REG_ITMP3);
4421
4422         /* save return value */
4423
4424         if (md->returntype.type != TYPE_VOID) {
4425                 if (IS_INT_LNG_TYPE(md->returntype.type))
4426                         M_LST(REG_RESULT, REG_SP, 0 * 8);
4427                 else
4428                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
4429         }
4430
4431         /* remove native stackframe info */
4432
4433         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4434         M_MOV_IMM((ptrint) codegen_finish_native_call, REG_ITMP1);
4435         M_CALL(REG_ITMP1);
4436
4437         /* generate call trace */
4438
4439         if (runverbose) {
4440                 /* just restore the value we need, don't care about the other */
4441
4442                 if (md->returntype.type != TYPE_VOID) {
4443                         if (IS_INT_LNG_TYPE(md->returntype.type))
4444                                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
4445                         else
4446                                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4447                 }
4448
4449                 M_MOV_IMM((ptrint) m, rd->argintregs[0]);
4450                 M_MOV(REG_RESULT, rd->argintregs[1]);
4451                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4452                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4453
4454                 M_MOV_IMM((ptrint) builtin_displaymethodstop, REG_ITMP1);
4455                 M_CALL(REG_ITMP1);
4456         }
4457
4458         /* check for exception */
4459
4460 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4461         M_MOV_IMM((ptrint) builtin_get_exceptionptrptr, REG_ITMP3);
4462         M_CALL(REG_ITMP3);
4463 #else
4464         M_MOV_IMM((ptrint) &_no_threads_exceptionptr, REG_RESULT);
4465 #endif
4466         M_ALD(REG_ITMP2, REG_RESULT, 0);
4467
4468         /* restore return value */
4469
4470         if (md->returntype.type != TYPE_VOID) {
4471                 if (IS_INT_LNG_TYPE(md->returntype.type))
4472                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
4473                 else
4474                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4475         }
4476
4477         /* test for exception */
4478
4479         M_TEST(REG_ITMP2);
4480         M_BNE(7 + 1);
4481
4482         /* remove stackframe */
4483
4484         M_AADD_IMM(stackframesize * 8, REG_SP);
4485         M_RET;
4486
4487
4488         /* handle exception */
4489
4490 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4491         M_LST(REG_ITMP2, REG_SP, 0 * 8);
4492         M_MOV_IMM((ptrint) builtin_get_exceptionptrptr, REG_ITMP3);
4493         M_CALL(REG_ITMP3);
4494         M_AST_IMM32(0, REG_RESULT, 0);                 /* clear exception pointer */
4495         M_LLD(REG_ITMP1_XPTR, REG_SP, 0 * 8);
4496 #else
4497         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
4498         M_MOV_IMM((ptrint) &_no_threads_exceptionptr, REG_ITMP3);
4499         M_AST_IMM32(0, REG_ITMP3, 0);                  /* clear exception pointer */
4500 #endif
4501
4502         /* remove stackframe */
4503
4504         M_AADD_IMM(stackframesize * 8, REG_SP);
4505
4506         M_LLD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
4507         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
4508
4509         M_MOV_IMM((ptrint) asm_handle_nat_exception, REG_ITMP3);
4510         M_JMP(REG_ITMP3);
4511
4512
4513         /* process patcher calls **************************************************/
4514
4515         {
4516                 u1          *xcodeptr;
4517                 patchref    *pref;
4518                 codegendata *tmpcd;
4519                 ptrint       mcode;
4520 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4521                 s4           disp;
4522 #endif
4523
4524                 tmpcd = DNEW(codegendata);
4525
4526                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4527                         /* Get machine code which is patched back in later. A             */
4528                         /* `call rel32' is 5 bytes long (but read 8 bytes).               */
4529
4530                         xcodeptr = cd->mcodebase + pref->branchpos;
4531                         mcode = *((ptrint *) xcodeptr);
4532
4533                         /* patch in `call rel32' to call the following code               */
4534
4535                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
4536                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
4537
4538                         /* move pointer to java_objectheader onto stack */
4539
4540 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4541                         /* create a virtual java_objectheader */
4542
4543                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4544                         disp = dseg_addaddress(cd, NULL);                   /* vftbl      */
4545
4546                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP3);
4547                         M_PUSH(REG_ITMP3);
4548 #else
4549                         M_PUSH_IMM(0);
4550 #endif
4551
4552                         /* move machine code bytes and classinfo pointer into registers */
4553
4554                         M_MOV_IMM((ptrint) mcode, REG_ITMP3);
4555                         M_PUSH(REG_ITMP3);
4556                         M_MOV_IMM((ptrint) pref->ref, REG_ITMP3);
4557                         M_PUSH(REG_ITMP3);
4558                         M_MOV_IMM((ptrint) pref->disp, REG_ITMP3);
4559                         M_PUSH(REG_ITMP3);
4560
4561                         M_MOV_IMM((ptrint) pref->patcher, REG_ITMP3);
4562                         M_PUSH(REG_ITMP3);
4563
4564                         M_MOV_IMM((ptrint) asm_wrapper_patcher, REG_ITMP3);
4565                         M_JMP(REG_ITMP3);
4566                 }
4567         }
4568
4569         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
4570
4571         return m->entrypoint;
4572 }
4573
4574
4575 /*
4576  * These are local overrides for various environment variables in Emacs.
4577  * Please do not remove this and leave it at the end of the file, where
4578  * Emacs will automagically detect them.
4579  * ---------------------------------------------------------------------
4580  * Local variables:
4581  * mode: c
4582  * indent-tabs-mode: t
4583  * c-basic-offset: 4
4584  * tab-width: 4
4585  * End:
4586  */