src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005 R. Grafl, A. Krall, C. Kruegel, C. Oates,
   4    R. Obermaisser, M. Platter, M. Probst, S. Ring, E. Steiner,
   5    C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich, J. Wenninger,
   6    Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  23    02111-1307, USA.
  24
  25    Contact: cacao@complang.tuwien.ac.at
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31
  32    $Id: codegen.c 3096 2005-07-21 14:01:02Z twisti $
  33
  34 */
  35
  36
  37 #include <stdio.h>
  38
  39 #include "config.h"
  40
  41 #include "md.h"
  42 #include "md-abi.h"
  43 #include "md-abi.inc"
  44
  45 #include "vm/jit/x86_64/arch.h"
  46 #include "vm/jit/x86_64/codegen.h"
  47 #include "vm/jit/x86_64/emitfuncs.h"
  48 #include "vm/jit/x86_64/types.h"
  49 #include "vm/jit/x86_64/asmoffsets.h"
  50
  51 #include "cacao/cacao.h"
  52 #include "native/native.h"
  53 #include "vm/global.h"
  54 #include "vm/builtin.h"
  55 #include "vm/loader.h"
  56 #include "vm/statistics.h"
  57 #include "vm/stringlocal.h"
  58 #include "vm/tables.h"
  59 #include "vm/jit/asmpart.h"
  60 #include "vm/jit/codegen.inc"
  61 #include "vm/jit/helper.h"
  62 #include "vm/jit/jit.h"
  63
  64 #if defined(LSRA)
  65 # include "vm/jit/lsra.inc"
  66 #endif
  67
  68 #include "vm/jit/parse.h"
  69 #include "vm/jit/patcher.h"
  70 #include "vm/jit/reg.h"
  71 #include "vm/jit/reg.inc"
  72
  73
  74 /* codegen *********************************************************************
  75
  76    Generates machine code.
  77
  78 *******************************************************************************/
  79
  80 void codegen(methodinfo *m, codegendata *cd, registerdata *rd)
  81 {
  82         s4                  len, s1, s2, s3, d, disp;
  83         u2                  currentline;
  84         ptrint              a;
  85         s4                  parentargs_base;
  86         stackptr            src;
  87         varinfo            *var;
  88         basicblock         *bptr;
  89         instruction        *iptr;
  90         exceptiontable     *ex;
  91         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
  92         builtintable_entry *bte;
  93         methoddesc         *md;
  94
  95         /* prevent compiler warnings */
  96
  97         d = 0;
  98         lm = NULL;
  99         bte = NULL;
 100
 101         {
 102         s4 i, p, t, l;
 103         s4 savedregs_num;
 104
 105         savedregs_num = 0;
 106
 107         /* space to save used callee saved registers */
 108
 109         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 110         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 111
 112         parentargs_base = rd->memuse + savedregs_num;
 113
 114 #if defined(USE_THREADS)
 115         /* space to save argument of monitor_enter */
 116
 117         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 118                 parentargs_base++;
 119 #endif
 120
 121     /* Keep stack of non-leaf functions 16-byte aligned for calls into native */
 122         /* code e.g. libc or jni (alignment problems with movaps).                */
 123
 124         if (!m->isleafmethod || runverbose)
 125                 parentargs_base |= 0x1;
 126
 127         /* create method header */
 128
 129         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 130         (void) dseg_adds4(cd, parentargs_base * 8);             /* FrameSize      */
 131
 132 #if defined(USE_THREADS)
 133         /* IsSync contains the offset relative to the stack pointer for the
 134            argument of monitor_exit used in the exception handler. Since the
 135            offset could be zero and give a wrong meaning of the flag it is
 136            offset by one.
 137         */
 138
 139         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 140                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);        /* IsSync         */
 141         else
 142 #endif
 143                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 144
 145         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 146         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse);  /* IntSave        */
 147         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse);  /* FltSave        */
 148
 149         (void) dseg_addlinenumbertablesize(cd);
 150
 151         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 152
 153         /* create exception table */
 154
 155         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 156                 dseg_addtarget(cd, ex->start);
 157                 dseg_addtarget(cd, ex->end);
 158                 dseg_addtarget(cd, ex->handler);
 159                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 160         }
 161
 162         /* initialize mcode variables */
 163
 164         cd->mcodeptr = (u1 *) cd->mcodebase;
 165         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
 166         MCODECHECK(512);
 167
 168         /* create stack frame (if necessary) */
 169
 170         if (parentargs_base)
 171                 M_ASUB_IMM(parentargs_base * 8, REG_SP);
 172
 173         /* save used callee saved registers */
 174
 175         p = parentargs_base;
 176         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 177                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 178         }
 179         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 180                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 181         }
 182
 183         /* take arguments out of register or stack frame */
 184
 185         md = m->parseddesc;
 186
 187         for (p = 0, l = 0; p < md->paramcount; p++) {
 188                 t = md->paramtypes[p].type;
 189                 var = &(rd->locals[l][t]);
 190                 l++;
 191                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 192                         l++;
 193                 if (var->type < 0)
 194                         continue;
 195                 s1 = md->params[p].regoff;
 196                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 197                         s2 = rd->argintregs[s1];
 198                         if (!md->params[p].inmemory) {           /* register arguments    */
 199                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 200                                         M_INTMOVE(s2, var->regoff);
 201
 202                                 } else {                             /* reg arg -> spilled    */
 203                                     M_LST(s2, REG_SP, var->regoff * 8);
 204                                 }
 205
 206                         } else {                                 /* stack arguments       */
 207                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 208                                         /* + 8 for return address */
 209                                         M_LLD(var->regoff, REG_SP, (parentargs_base + s1) * 8 + 8);
 210
 211                                 } else {                             /* stack arg -> spilled  */
 212                                         var->regoff = parentargs_base + s1 + 1;
 213                                 }
 214                         }
 215
 216                 } else {                                     /* floating args         */
 217                         if (!md->params[p].inmemory) {           /* register arguments    */
 218                                 s2 = rd->argfltregs[s1];
 219                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 220                                         M_FLTMOVE(s2, var->regoff);
 221
 222                                 } else {                                         /* reg arg -> spilled    */
 223                                         M_DST(s2, REG_SP, var->regoff * 8);
 224                                 }
 225
 226                         } else {                                 /* stack arguments       */
 227                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 228                                         M_DLD(var->regoff, REG_SP, (parentargs_base + s1) * 8 + 8);
 229
 230                                 } else {
 231                                         var->regoff = parentargs_base + s1 + 1;
 232                                 }
 233                         }
 234                 }
 235         }  /* end for */
 236
 237         /* save monitorenter argument */
 238
 239 #if defined(USE_THREADS)
 240         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 241                 /* stack offset for monitor argument */
 242
 243                 s1 = rd->memuse;
 244
 245                 if (runverbose) {
 246                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 247
 248                         for (p = 0; p < INT_ARG_CNT; p++)
 249                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 250
 251                         for (p = 0; p < FLT_ARG_CNT; p++)
 252                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 253
 254                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 255                 }
 256
 257                 /* decide which monitor enter function to call */
 258
 259                 if (m->flags & ACC_STATIC) {
 260                         x86_64_mov_imm_reg(cd, (ptrint) m->class, REG_ITMP1);
 261                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, s1 * 8);
 262                         M_INTMOVE(REG_ITMP1, rd->argintregs[0]);
 263                         x86_64_mov_imm_reg(cd, (ptrint) BUILTIN_staticmonitorenter, REG_ITMP1);
 264                         x86_64_call_reg(cd, REG_ITMP1);
 265
 266                 } else {
 267                         x86_64_test_reg_reg(cd, rd->argintregs[0], rd->argintregs[0]);
 268                         x86_64_jcc(cd, X86_64_CC_Z, 0);
 269                         codegen_addxnullrefs(cd, cd->mcodeptr);
 270                         x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, s1 * 8);
 271                         x86_64_mov_imm_reg(cd, (ptrint) BUILTIN_monitorenter, REG_ITMP1);
 272                         x86_64_call_reg(cd, REG_ITMP1);
 273                 }
 274
 275                 if (runverbose) {
 276                         for (p = 0; p < INT_ARG_CNT; p++)
 277                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 278
 279                         for (p = 0; p < FLT_ARG_CNT; p++)
 280                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 281
 282                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 283                 }
 284         }
 285 #endif
 286
 287         /* Copy argument registers to stack and call trace function with pointer  */
 288         /* to arguments on stack.                                                 */
 289
 290         if (runverbose || opt_stat) {
 291                 M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 292
 293                 /* save integer argument registers */
 294
 295                 for (p = 0; p < INT_ARG_CNT; p++)
 296                         M_LST(rd->argintregs[p], REG_SP, (1 + p) * 8);
 297
 298                 /* save float argument registers */
 299
 300                 for (p = 0; p < FLT_ARG_CNT; p++)
 301                         M_DST(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 302
 303                 /* save temporary registers for leaf methods */
 304
 305                 if (m->isleafmethod) {
 306                         for (p = 0; p < INT_TMP_CNT; p++)
 307                                 M_LST(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 308
 309                         for (p = 0; p < FLT_TMP_CNT; p++)
 310                                 M_DST(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 311                 }
 312
 313                 if (runverbose) {
 314                         /* show integer hex code for float arguments */
 315
 316                         for (p = 0, l = 0; p < md->paramcount && p < INT_ARG_CNT; p++) {
 317                                 /* if the paramtype is a float, we have to right shift all    */
 318                                 /* following integer registers                                */
 319
 320                                 if (IS_FLT_DBL_TYPE(md->paramtypes[p].type)) {
 321                                         for (s1 = INT_ARG_CNT - 2; s1 >= p; s1--) {
 322                                                 M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
 323                                         }
 324
 325                                         x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 326                                         l++;
 327                                 }
 328                         }
 329
 330                         x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP2);
 331                         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_SP, 0 * 8);
 332                         x86_64_mov_imm_reg(cd, (ptrint) builtin_trace_args, REG_ITMP1);
 333                         x86_64_call_reg(cd, REG_ITMP1);
 334                 }
 335                 STATS({
 336                         x86_64_mov_imm_reg(cd, (ptrint) compiledinvokation, REG_ITMP1);
 337                         x86_64_call_reg(cd, REG_ITMP1);
 338                 })
 339
 340                 /* restore integer argument registers */
 341
 342                 for (p = 0; p < INT_ARG_CNT; p++)
 343                         M_LLD(rd->argintregs[p], REG_SP, (1 + p) * 8);
 344
 345                 /* restore float argument registers */
 346
 347                 for (p = 0; p < FLT_ARG_CNT; p++)
 348                         M_DLD(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 349
 350                 /* restore temporary registers for leaf methods */
 351
 352                 if (m->isleafmethod) {
 353                         for (p = 0; p < INT_TMP_CNT; p++)
 354                                 M_LLD(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 355
 356                         for (p = 0; p < FLT_TMP_CNT; p++)
 357                                 M_DLD(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 358                 }
 359
 360                 M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 361         }
 362
 363         }
 364
 365         /* end of header generation */
 366
 367         /* walk through all basic blocks */
 368         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 369
 370                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 371
 372                 if (bptr->flags >= BBREACHED) {
 373
 374                         /* branch resolving */
 375
 376                         branchref *bref;
 377                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 378                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 379                                                                   bref->branchpos,
 380                                                                   bptr->mpc);
 381                         }
 382
 383                 /* copy interface registers to their destination */
 384
 385                 src = bptr->instack;
 386                 len = bptr->indepth;
 387                 MCODECHECK(512);
 388
 389 #ifdef LSRA
 390                 if (opt_lsra) {
 391                         while (src != NULL) {
 392                                 len--;
 393                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 394                                         if (bptr->type == BBTYPE_SBR) {
 395                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 396                                                 if (!(src->flags & INMEMORY))
 397                                                         d= src->regoff;
 398                                                 else
 399                                                         d=REG_ITMP1;
 400                                                 x86_64_pop_reg(cd, d);
 401                                                 store_reg_to_var_int(src, d);
 402
 403                                         } else if (bptr->type == BBTYPE_EXH) {
 404                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 405                                                 if (!(src->flags & INMEMORY))
 406                                                         d= src->regoff;
 407                                                 else
 408                                                         d=REG_ITMP1;
 409                                                 M_INTMOVE(REG_ITMP1, d);
 410                                                 store_reg_to_var_int(src, d);
 411                                         }
 412                                 }
 413                                 src = src->prev;
 414                         }
 415
 416                 } else {
 417 #endif
 418
 419         while (src != NULL) {
 420                         len--;
 421                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 422                                 if (bptr->type == BBTYPE_SBR) {
 423                                         d = reg_of_var(rd, src, REG_ITMP1);
 424                                         x86_64_pop_reg(cd, d);
 425                                         store_reg_to_var_int(src, d);
 426
 427                                 } else if (bptr->type == BBTYPE_EXH) {
 428                                         d = reg_of_var(rd, src, REG_ITMP1);
 429                                         M_INTMOVE(REG_ITMP1, d);
 430                                         store_reg_to_var_int(src, d);
 431                                 }
 432
 433                         } else {
 434                                 d = reg_of_var(rd, src, REG_ITMP1);
 435                                 if ((src->varkind != STACKVAR)) {
 436                                         s2 = src->type;
 437                                         if (IS_FLT_DBL_TYPE(s2)) {
 438                                                 s1 = rd->interfaces[len][s2].regoff;
 439                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 440                                                         M_FLTMOVE(s1, d);
 441
 442                                                 } else {
 443                                                         x86_64_movq_membase_reg(cd, REG_SP, s1 * 8, d);
 444                                                 }
 445                                                 store_reg_to_var_flt(src, d);
 446
 447                                         } else {
 448                                                 s1 = rd->interfaces[len][s2].regoff;
 449                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 450                                                         M_INTMOVE(s1, d);
 451
 452                                                 } else {
 453                                                         x86_64_mov_membase_reg(cd, REG_SP, s1 * 8, d);
 454                                                 }
 455                                                 store_reg_to_var_int(src, d);
 456                                         }
 457                                 }
 458                         }
 459                         src = src->prev;
 460                 }
 461 #ifdef LSRA
 462                 }
 463 #endif
 464                 /* walk through all instructions */
 465
 466                 src = bptr->instack;
 467                 len = bptr->icount;
 468                 currentline=0;
 469
 470                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 471                         if (iptr->line != currentline) {
 472                                 dseg_addlinenumber(cd, iptr->line, cd->mcodeptr);
 473                                 currentline = iptr->line;
 474                         }
 475
 476                         MCODECHECK(1024);                         /* 1KB should be enough */
 477
 478                         switch (iptr->opc) {
 479                         case ICMD_INLINE_START: /* internal ICMDs                         */
 480                         case ICMD_INLINE_END:
 481                                 break;
 482
 483                         case ICMD_NOP:    /* ...  ==> ...                                 */
 484                                 break;
 485
 486                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 487                                 if (src->flags & INMEMORY) {
 488                                         x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
 489
 490                                 } else {
 491                                         x86_64_test_reg_reg(cd, src->regoff, src->regoff);
 492                                 }
 493                                 x86_64_jcc(cd, X86_64_CC_Z, 0);
 494                                 codegen_addxnullrefs(cd, cd->mcodeptr);
 495                                 break;
 496
 497                 /* constant operations ************************************************/
 498
 499                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 500                                       /* op1 = 0, val.i = constant                    */
 501
 502                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 503                         if (iptr->val.i == 0) {
 504                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 505                         } else {
 506                                 x86_64_movl_imm_reg(cd, iptr->val.i, d);
 507                         }
 508                         store_reg_to_var_int(iptr->dst, d);
 509                         break;
 510
 511                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 512                                       /* op1 = 0, val.a = constant                    */
 513
 514                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 515                         if (iptr->val.a == 0) {
 516                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 517                         } else {
 518                                 x86_64_mov_imm_reg(cd, (s8) iptr->val.a, d);
 519                         }
 520                         store_reg_to_var_int(iptr->dst, d);
 521                         break;
 522
 523                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 524                                       /* op1 = 0, val.l = constant                    */
 525
 526                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 527                         if (iptr->val.l == 0) {
 528                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 529                         } else {
 530                                 x86_64_mov_imm_reg(cd, iptr->val.l, d);
 531                         }
 532                         store_reg_to_var_int(iptr->dst, d);
 533                         break;
 534
 535                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 536                                       /* op1 = 0, val.f = constant                    */
 537
 538                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 539                         disp = dseg_addfloat(cd, iptr->val.f);
 540                         x86_64_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + disp, d);
 541                         store_reg_to_var_flt(iptr->dst, d);
 542                         break;
 543
 544                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 545                                       /* op1 = 0, val.d = constant                    */
 546
 547                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 548                         disp = dseg_adddouble(cd, iptr->val.d);
 549                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, d);
 550                         store_reg_to_var_flt(iptr->dst, d);
 551                         break;
 552
 553
 554                 /* load/store operations **********************************************/
 555
 556                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 557                                       /* op1 = local variable                         */
 558
 559                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 560                         if ((iptr->dst->varkind == LOCALVAR) &&
 561                             (iptr->dst->varnum == iptr->op1)) {
 562                                 break;
 563                         }
 564                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 565                         if (var->flags & INMEMORY) {
 566                                 x86_64_movl_membase_reg(cd, REG_SP, var->regoff * 8, d);
 567                                 store_reg_to_var_int(iptr->dst, d);
 568
 569                         } else {
 570                                 if (iptr->dst->flags & INMEMORY) {
 571                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 572
 573                                 } else {
 574                                         M_INTMOVE(var->regoff, d);
 575                                 }
 576                         }
 577                         break;
 578
 579                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 580                 case ICMD_ALOAD:      /* op1 = local variable                         */
 581
 582                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 583                         if ((iptr->dst->varkind == LOCALVAR) &&
 584                             (iptr->dst->varnum == iptr->op1)) {
 585                                 break;
 586                         }
 587                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 588                         if (var->flags & INMEMORY) {
 589                                 x86_64_mov_membase_reg(cd, REG_SP, var->regoff * 8, d);
 590                                 store_reg_to_var_int(iptr->dst, d);
 591
 592                         } else {
 593                                 if (iptr->dst->flags & INMEMORY) {
 594                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 595
 596                                 } else {
 597                                         M_INTMOVE(var->regoff, d);
 598                                 }
 599                         }
 600                         break;
 601
 602                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 603                 case ICMD_DLOAD:      /* op1 = local variable                         */
 604
 605                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 606                         if ((iptr->dst->varkind == LOCALVAR) &&
 607                             (iptr->dst->varnum == iptr->op1)) {
 608                                 break;
 609                         }
 610                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 611                         if (var->flags & INMEMORY) {
 612                                 x86_64_movq_membase_reg(cd, REG_SP, var->regoff * 8, d);
 613                                 store_reg_to_var_flt(iptr->dst, d);
 614
 615                         } else {
 616                                 if (iptr->dst->flags & INMEMORY) {
 617                                         x86_64_movq_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 618
 619                                 } else {
 620                                         M_FLTMOVE(var->regoff, d);
 621                                 }
 622                         }
 623                         break;
 624
 625                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 626                 case ICMD_LSTORE:     /* op1 = local variable                         */
 627                 case ICMD_ASTORE:
 628
 629                         if ((src->varkind == LOCALVAR) &&
 630                             (src->varnum == iptr->op1)) {
 631                                 break;
 632                         }
 633                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 634                         if (var->flags & INMEMORY) {
 635                                 var_to_reg_int(s1, src, REG_ITMP1);
 636                                 x86_64_mov_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 637
 638                         } else {
 639                                 var_to_reg_int(s1, src, var->regoff);
 640                                 M_INTMOVE(s1, var->regoff);
 641                         }
 642                         break;
 643
 644                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 645                 case ICMD_DSTORE:     /* op1 = local variable                         */
 646
 647                         if ((src->varkind == LOCALVAR) &&
 648                             (src->varnum == iptr->op1)) {
 649                                 break;
 650                         }
 651                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 652                         if (var->flags & INMEMORY) {
 653                                 var_to_reg_flt(s1, src, REG_FTMP1);
 654                                 x86_64_movq_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 655
 656                         } else {
 657                                 var_to_reg_flt(s1, src, var->regoff);
 658                                 M_FLTMOVE(s1, var->regoff);
 659                         }
 660                         break;
 661
 662
 663                 /* pop/dup/swap operations ********************************************/
 664
 665                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 666
 667                 case ICMD_POP:        /* ..., value  ==> ...                          */
 668                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 669                         break;
 670
 671                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 672                         M_COPY(src, iptr->dst);
 673                         break;
 674
 675                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 676
 677                         M_COPY(src,       iptr->dst);
 678                         M_COPY(src->prev, iptr->dst->prev);
 679                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 680                         break;
 681
 682                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 683
 684                         M_COPY(src,             iptr->dst);
 685                         M_COPY(src->prev,       iptr->dst->prev);
 686                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 687                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 688                         break;
 689
 690                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 691
 692                         M_COPY(src,       iptr->dst);
 693                         M_COPY(src->prev, iptr->dst->prev);
 694                         break;
 695
 696                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 697
 698                         M_COPY(src,             iptr->dst);
 699                         M_COPY(src->prev,       iptr->dst->prev);
 700                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 701                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 702                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 703                         break;
 704
 705                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 706
 707                         M_COPY(src,                   iptr->dst);
 708                         M_COPY(src->prev,             iptr->dst->prev);
 709                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 710                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 711                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 712                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 713                         break;
 714
 715                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 716
 717                         M_COPY(src,       iptr->dst->prev);
 718                         M_COPY(src->prev, iptr->dst);
 719                         break;
 720
 721
 722                 /* integer operations *************************************************/
 723
 724                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 725
 726                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 727                         if (iptr->dst->flags & INMEMORY) {
 728                                 if (src->flags & INMEMORY) {
 729                                         if (src->regoff == iptr->dst->regoff) {
 730                                                 x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 731
 732                                         } else {
 733                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 734                                                 x86_64_negl_reg(cd, REG_ITMP1);
 735                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 736                                         }
 737
 738                                 } else {
 739                                         x86_64_movl_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 740                                         x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 741                                 }
 742
 743                         } else {
 744                                 if (src->flags & INMEMORY) {
 745                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 746                                         x86_64_negl_reg(cd, d);
 747
 748                                 } else {
 749                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 750                                         x86_64_negl_reg(cd, iptr->dst->regoff);
 751                                 }
 752                         }
 753                         break;
 754
 755                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 756
 757                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 758                         if (iptr->dst->flags & INMEMORY) {
 759                                 if (src->flags & INMEMORY) {
 760                                         if (src->regoff == iptr->dst->regoff) {
 761                                                 x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 762
 763                                         } else {
 764                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 765                                                 x86_64_neg_reg(cd, REG_ITMP1);
 766                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 767                                         }
 768
 769                                 } else {
 770                                         x86_64_mov_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 771                                         x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 772                                 }
 773
 774                         } else {
 775                                 if (src->flags & INMEMORY) {
 776                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 777                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 778
 779                                 } else {
 780                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 781                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 782                                 }
 783                         }
 784                         break;
 785
 786                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 787
 788                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 789                         if (src->flags & INMEMORY) {
 790                                 x86_64_movslq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 791
 792                         } else {
 793                                 x86_64_movslq_reg_reg(cd, src->regoff, d);
 794                         }
 795                         store_reg_to_var_int(iptr->dst, d);
 796                         break;
 797
 798                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 799
 800                         var_to_reg_int(s1, src, REG_ITMP1);
 801                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 802                         M_INTMOVE(s1, d);
 803                         store_reg_to_var_int(iptr->dst, d);
 804                         break;
 805
 806                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 807
 808                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 809                         if (src->flags & INMEMORY) {
 810                                 x86_64_movsbq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 811
 812                         } else {
 813                                 x86_64_movsbq_reg_reg(cd, src->regoff, d);
 814                         }
 815                         store_reg_to_var_int(iptr->dst, d);
 816                         break;
 817
 818                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 819
 820                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 821                         if (src->flags & INMEMORY) {
 822                                 x86_64_movzwq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 823
 824                         } else {
 825                                 x86_64_movzwq_reg_reg(cd, src->regoff, d);
 826                         }
 827                         store_reg_to_var_int(iptr->dst, d);
 828                         break;
 829
 830                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 831
 832                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 833                         if (src->flags & INMEMORY) {
 834                                 x86_64_movswq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 835
 836                         } else {
 837                                 x86_64_movswq_reg_reg(cd, src->regoff, d);
 838                         }
 839                         store_reg_to_var_int(iptr->dst, d);
 840                         break;
 841
 842
 843                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 844
 845                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 846                         x86_64_emit_ialu(cd, X86_64_ADD, src, iptr);
 847                         break;
 848
 849                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 850                                       /* val.i = constant                             */
 851
 852                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 853                         x86_64_emit_ialuconst(cd, X86_64_ADD, src, iptr);
 854                         break;
 855
 856                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 857
 858                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 859                         x86_64_emit_lalu(cd, X86_64_ADD, src, iptr);
 860                         break;
 861
 862                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 863                                       /* val.l = constant                             */
 864
 865                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 866                         x86_64_emit_laluconst(cd, X86_64_ADD, src, iptr);
 867                         break;
 868
 869                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 870
 871                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 872                         if (iptr->dst->flags & INMEMORY) {
 873                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 874                                         if (src->prev->regoff == iptr->dst->regoff) {
 875                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 876                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 877
 878                                         } else {
 879                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 880                                                 x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 881                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 882                                         }
 883
 884                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 885                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 886                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 887                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 888
 889                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 890                                         if (src->prev->regoff == iptr->dst->regoff) {
 891                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 892
 893                                         } else {
 894                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 895                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 896                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 897                                         }
 898
 899                                 } else {
 900                                         x86_64_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 901                                         x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 902                                 }
 903
 904                         } else {
 905                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 906                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 907                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 908
 909                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 910                                         M_INTMOVE(src->prev->regoff, d);
 911                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 912
 913                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 914                                         /* workaround for reg alloc */
 915                                         if (src->regoff == iptr->dst->regoff) {
 916                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 917                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 918                                                 M_INTMOVE(REG_ITMP1, d);
 919
 920                                         } else {
 921                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 922                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 923                                         }
 924
 925                                 } else {
 926                                         /* workaround for reg alloc */
 927                                         if (src->regoff == iptr->dst->regoff) {
 928                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
 929                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 930                                                 M_INTMOVE(REG_ITMP1, d);
 931
 932                                         } else {
 933                                                 M_INTMOVE(src->prev->regoff, d);
 934                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 935                                         }
 936                                 }
 937                         }
 938                         break;
 939
 940                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 941                                       /* val.i = constant                             */
 942
 943                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 944                         x86_64_emit_ialuconst(cd, X86_64_SUB, src, iptr);
 945                         break;
 946
 947                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 948
 949                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 950                         if (iptr->dst->flags & INMEMORY) {
 951                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 952                                         if (src->prev->regoff == iptr->dst->regoff) {
 953                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 954                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 955
 956                                         } else {
 957                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 958                                                 x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 959                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 960                                         }
 961
 962                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 963                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 964                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 965                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 966
 967                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 968                                         if (src->prev->regoff == iptr->dst->regoff) {
 969                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 970
 971                                         } else {
 972                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 973                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 974                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 975                                         }
 976
 977                                 } else {
 978                                         x86_64_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 979                                         x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 980                                 }
 981
 982                         } else {
 983                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 984                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 985                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 986
 987                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 988                                         M_INTMOVE(src->prev->regoff, d);
 989                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 990
 991                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 992                                         /* workaround for reg alloc */
 993                                         if (src->regoff == iptr->dst->regoff) {
 994                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 995                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 996                                                 M_INTMOVE(REG_ITMP1, d);
 997
 998                                         } else {
 999                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1000                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1001                                         }
1002
1003                                 } else {
1004                                         /* workaround for reg alloc */
1005                                         if (src->regoff == iptr->dst->regoff) {
1006                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1007                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1008                                                 M_INTMOVE(REG_ITMP1, d);
1009
1010                                         } else {
1011                                                 M_INTMOVE(src->prev->regoff, d);
1012                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1013                                         }
1014                                 }
1015                         }
1016                         break;
1017
1018                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1019                                       /* val.l = constant                             */
1020
1021                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1022                         x86_64_emit_laluconst(cd, X86_64_SUB, src, iptr);
1023                         break;
1024
1025                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1026
1027                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1028                         if (iptr->dst->flags & INMEMORY) {
1029                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1030                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1031                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1032                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1033
1034                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1035                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1036                                         x86_64_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1037                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1038
1039                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1040                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1041                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1042                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1043
1044                                 } else {
1045                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1046                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1047                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1048                                 }
1049
1050                         } else {
1051                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1052                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1053                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1054
1055                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1056                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1057                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1058
1059                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1060                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1061                                         x86_64_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1062
1063                                 } else {
1064                                         if (src->regoff == iptr->dst->regoff) {
1065                                                 x86_64_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1066
1067                                         } else {
1068                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1069                                                 x86_64_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1070                                         }
1071                                 }
1072                         }
1073                         break;
1074
1075                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1076                                       /* val.i = constant                             */
1077
1078                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1079                         if (iptr->dst->flags & INMEMORY) {
1080                                 if (src->flags & INMEMORY) {
1081                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1082                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1083
1084                                 } else {
1085                                         x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1086                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1087                                 }
1088
1089                         } else {
1090                                 if (src->flags & INMEMORY) {
1091                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1092
1093                                 } else {
1094                                         if (iptr->val.i == 2) {
1095                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1096                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1097
1098                                         } else {
1099                                                 x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1100                                         }
1101                                 }
1102                         }
1103                         break;
1104
1105                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1106
1107                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1108                         if (iptr->dst->flags & INMEMORY) {
1109                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1110                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1111                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1112                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1113
1114                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1115                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1116                                         x86_64_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1117                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1118
1119                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1120                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1121                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1122                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1123
1124                                 } else {
1125                                         x86_64_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1126                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1127                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1128                                 }
1129
1130                         } else {
1131                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1132                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1133                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1134
1135                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1136                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1137                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1138
1139                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1140                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1141                                         x86_64_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1142
1143                                 } else {
1144                                         if (src->regoff == iptr->dst->regoff) {
1145                                                 x86_64_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1146
1147                                         } else {
1148                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1149                                                 x86_64_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1150                                         }
1151                                 }
1152                         }
1153                         break;
1154
1155                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1156                                       /* val.l = constant                             */
1157
1158                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1159                         if (iptr->dst->flags & INMEMORY) {
1160                                 if (src->flags & INMEMORY) {
1161                                         if (IS_IMM32(iptr->val.l)) {
1162                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1163
1164                                         } else {
1165                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1166                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1167                                         }
1168                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1169
1170                                 } else {
1171                                         if (IS_IMM32(iptr->val.l)) {
1172                                                 x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1173
1174                                         } else {
1175                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1176                                                 x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1177                                         }
1178                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1179                                 }
1180
1181                         } else {
1182                                 if (src->flags & INMEMORY) {
1183                                         if (IS_IMM32(iptr->val.l)) {
1184                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1185
1186                                         } else {
1187                                                 x86_64_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1188                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1189                                         }
1190
1191                                 } else {
1192                                         /* should match in many cases */
1193                                         if (iptr->val.l == 2) {
1194                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1195                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1196
1197                                         } else {
1198                                                 if (IS_IMM32(iptr->val.l)) {
1199                                                         x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1200
1201                                                 } else {
1202                                                         x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1203                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1204                                                         x86_64_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1205                                                 }
1206                                         }
1207                                 }
1208                         }
1209                         break;
1210
1211                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1212
1213                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1214                 if (src->prev->flags & INMEMORY) {
1215                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1216
1217                         } else {
1218                                 M_INTMOVE(src->prev->regoff, RAX);
1219                         }
1220
1221                         if (src->flags & INMEMORY) {
1222                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1223
1224                         } else {
1225                                 M_INTMOVE(src->regoff, REG_ITMP3);
1226                         }
1227                         gen_div_check(src);
1228
1229                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1230                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1231                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1232                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                /* 6 bytes */
1233
1234                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1235                         x86_64_cltd(cd);
1236                         x86_64_idivl_reg(cd, REG_ITMP3);
1237
1238                         if (iptr->dst->flags & INMEMORY) {
1239                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1240                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1241
1242                         } else {
1243                                 M_INTMOVE(RAX, iptr->dst->regoff);
1244
1245                                 if (iptr->dst->regoff != RDX) {
1246                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1247                                 }
1248                         }
1249                         break;
1250
1251                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1252                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1253                         if (src->prev->flags & INMEMORY) {
1254                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1255
1256                         } else {
1257                                 M_INTMOVE(src->prev->regoff, RAX);
1258                         }
1259
1260                         if (src->flags & INMEMORY) {
1261                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1262
1263                         } else {
1264                                 M_INTMOVE(src->regoff, REG_ITMP3);
1265                         }
1266                         gen_div_check(src);
1267
1268                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1269
1270                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1271                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1272
1273
1274                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1275                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1276                         x86_64_jcc(cd, X86_64_CC_E, 1 + 3);                    /* 6 bytes */
1277
1278                         x86_64_cltd(cd);
1279                         x86_64_idivl_reg(cd, REG_ITMP3);
1280
1281                         if (iptr->dst->flags & INMEMORY) {
1282                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1283                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1284
1285                         } else {
1286                                 M_INTMOVE(RDX, iptr->dst->regoff);
1287
1288                                 if (iptr->dst->regoff != RDX) {
1289                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1290                                 }
1291                         }
1292                         break;
1293
1294                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1295                                       /* val.i = constant                             */
1296
1297                         var_to_reg_int(s1, src, REG_ITMP1);
1298                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1299                         M_INTMOVE(s1, REG_ITMP1);
1300                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1301                         x86_64_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1302                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1303                         x86_64_shiftl_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1304                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1305                         store_reg_to_var_int(iptr->dst, d);
1306                         break;
1307
1308                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1309                                       /* val.i = constant                             */
1310
1311                         var_to_reg_int(s1, src, REG_ITMP1);
1312                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1313                         M_INTMOVE(s1, REG_ITMP1);
1314                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1315                         x86_64_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1316                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1317                         x86_64_alul_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1318                         x86_64_alul_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1319                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1320                         store_reg_to_var_int(iptr->dst, d);
1321                         break;
1322
1323
1324                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1325
1326                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1327                 if (src->prev->flags & INMEMORY) {
1328                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1329
1330                         } else {
1331                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1332                         }
1333
1334                         if (src->flags & INMEMORY) {
1335                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1336
1337                         } else {
1338                                 M_INTMOVE(src->regoff, REG_ITMP3);
1339                         }
1340                         gen_div_check(src);
1341
1342                         x86_64_mov_imm_reg(cd, 0x8000000000000000LL, REG_ITMP2);    /* check as described in jvm spec */
1343                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
1344                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1345                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);          /* 4 bytes */
1346                         x86_64_jcc(cd, X86_64_CC_E, 3 + 2 + 3);                     /* 6 bytes */
1347
1348                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1349                         x86_64_cqto(cd);
1350                         x86_64_idiv_reg(cd, REG_ITMP3);
1351
1352                         if (iptr->dst->flags & INMEMORY) {
1353                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1354                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1355
1356                         } else {
1357                                 M_INTMOVE(RAX, iptr->dst->regoff);
1358
1359                                 if (iptr->dst->regoff != RDX) {
1360                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1361                                 }
1362                         }
1363                         break;
1364
1365                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1366
1367                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1368                         if (src->prev->flags & INMEMORY) {
1369                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1370
1371                         } else {
1372                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1373                         }
1374
1375                         if (src->flags & INMEMORY) {
1376                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1377
1378                         } else {
1379                                 M_INTMOVE(src->regoff, REG_ITMP3);
1380                         }
1381                         gen_div_check(src);
1382
1383                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1384
1385                         x86_64_mov_imm_reg(cd, 0x8000000000000000LL, REG_ITMP2);    /* check as described in jvm spec */
1386                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
1387                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1388
1389
1390                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);              /* 2 bytes */
1391                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);          /* 4 bytes */
1392                         x86_64_jcc(cd, X86_64_CC_E, 2 + 3);                         /* 6 bytes */
1393
1394                         x86_64_cqto(cd);
1395                         x86_64_idiv_reg(cd, REG_ITMP3);
1396
1397                         if (iptr->dst->flags & INMEMORY) {
1398                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1399                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1400
1401                         } else {
1402                                 M_INTMOVE(RDX, iptr->dst->regoff);
1403
1404                                 if (iptr->dst->regoff != RDX) {
1405                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1406                                 }
1407                         }
1408                         break;
1409
1410                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1411                                       /* val.i = constant                             */
1412
1413                         var_to_reg_int(s1, src, REG_ITMP1);
1414                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1415                         M_INTMOVE(s1, REG_ITMP1);
1416                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1417                         x86_64_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1418                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1419                         x86_64_shift_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1420                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1421                         store_reg_to_var_int(iptr->dst, d);
1422                         break;
1423
1424                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1425                                       /* val.l = constant                             */
1426
1427                         var_to_reg_int(s1, src, REG_ITMP1);
1428                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1429                         M_INTMOVE(s1, REG_ITMP1);
1430                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1431                         x86_64_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1432                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1433                         x86_64_alu_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1434                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1435                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1436                         store_reg_to_var_int(iptr->dst, d);
1437                         break;
1438
1439                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1440
1441                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1442                         x86_64_emit_ishift(cd, X86_64_SHL, src, iptr);
1443                         break;
1444
1445                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1446                                       /* val.i = constant                             */
1447
1448                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1449                         x86_64_emit_ishiftconst(cd, X86_64_SHL, src, iptr);
1450                         break;
1451
1452                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1453
1454                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1455                         x86_64_emit_ishift(cd, X86_64_SAR, src, iptr);
1456                         break;
1457
1458                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1459                                       /* val.i = constant                             */
1460
1461                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1462                         x86_64_emit_ishiftconst(cd, X86_64_SAR, src, iptr);
1463                         break;
1464
1465                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1466
1467                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1468                         x86_64_emit_ishift(cd, X86_64_SHR, src, iptr);
1469                         break;
1470
1471                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1472                                       /* val.i = constant                             */
1473
1474                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1475                         x86_64_emit_ishiftconst(cd, X86_64_SHR, src, iptr);
1476                         break;
1477
1478                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1479
1480                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1481                         x86_64_emit_lshift(cd, X86_64_SHL, src, iptr);
1482                         break;
1483
1484         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1485                                           /* val.i = constant                             */
1486
1487                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1488                         x86_64_emit_lshiftconst(cd, X86_64_SHL, src, iptr);
1489                         break;
1490
1491                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1492
1493                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1494                         x86_64_emit_lshift(cd, X86_64_SAR, src, iptr);
1495                         break;
1496
1497                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1498                                       /* val.i = constant                             */
1499
1500                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1501                         x86_64_emit_lshiftconst(cd, X86_64_SAR, src, iptr);
1502                         break;
1503
1504                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1505
1506                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1507                         x86_64_emit_lshift(cd, X86_64_SHR, src, iptr);
1508                         break;
1509
1510                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1511                                       /* val.l = constant                             */
1512
1513                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1514                         x86_64_emit_lshiftconst(cd, X86_64_SHR, src, iptr);
1515                         break;
1516
1517                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1518
1519                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1520                         x86_64_emit_ialu(cd, X86_64_AND, src, iptr);
1521                         break;
1522
1523                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1524                                       /* val.i = constant                             */
1525
1526                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1527                         x86_64_emit_ialuconst(cd, X86_64_AND, src, iptr);
1528                         break;
1529
1530                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1531
1532                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1533                         x86_64_emit_lalu(cd, X86_64_AND, src, iptr);
1534                         break;
1535
1536                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1537                                       /* val.l = constant                             */
1538
1539                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1540                         x86_64_emit_laluconst(cd, X86_64_AND, src, iptr);
1541                         break;
1542
1543                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1544
1545                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1546                         x86_64_emit_ialu(cd, X86_64_OR, src, iptr);
1547                         break;
1548
1549                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1550                                       /* val.i = constant                             */
1551
1552                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1553                         x86_64_emit_ialuconst(cd, X86_64_OR, src, iptr);
1554                         break;
1555
1556                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1557
1558                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1559                         x86_64_emit_lalu(cd, X86_64_OR, src, iptr);
1560                         break;
1561
1562                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1563                                       /* val.l = constant                             */
1564
1565                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1566                         x86_64_emit_laluconst(cd, X86_64_OR, src, iptr);
1567                         break;
1568
1569                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1570
1571                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1572                         x86_64_emit_ialu(cd, X86_64_XOR, src, iptr);
1573                         break;
1574
1575                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1576                                       /* val.i = constant                             */
1577
1578                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1579                         x86_64_emit_ialuconst(cd, X86_64_XOR, src, iptr);
1580                         break;
1581
1582                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1583
1584                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1585                         x86_64_emit_lalu(cd, X86_64_XOR, src, iptr);
1586                         break;
1587
1588                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1589                                       /* val.l = constant                             */
1590
1591                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1592                         x86_64_emit_laluconst(cd, X86_64_XOR, src, iptr);
1593                         break;
1594
1595
1596                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1597                                       /* op1 = variable, val.i = constant             */
1598
1599                         /* using inc and dec is definitely faster than add -- tested      */
1600                         /* with sieve                                                     */
1601
1602                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1603                         d = var->regoff;
1604                         if (var->flags & INMEMORY) {
1605                                 if (iptr->val.i == 1) {
1606                                         x86_64_incl_membase(cd, REG_SP, d * 8);
1607
1608                                 } else if (iptr->val.i == -1) {
1609                                         x86_64_decl_membase(cd, REG_SP, d * 8);
1610
1611                                 } else {
1612                                         x86_64_alul_imm_membase(cd, X86_64_ADD, iptr->val.i, REG_SP, d * 8);
1613                                 }
1614
1615                         } else {
1616                                 if (iptr->val.i == 1) {
1617                                         x86_64_incl_reg(cd, d);
1618
1619                                 } else if (iptr->val.i == -1) {
1620                                         x86_64_decl_reg(cd, d);
1621
1622                                 } else {
1623                                         x86_64_alul_imm_reg(cd, X86_64_ADD, iptr->val.i, d);
1624                                 }
1625                         }
1626                         break;
1627
1628
1629                 /* floating operations ************************************************/
1630
1631                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1632
1633                         var_to_reg_flt(s1, src, REG_FTMP1);
1634                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1635                         disp = dseg_adds4(cd, 0x80000000);
1636                         M_FLTMOVE(s1, d);
1637                         x86_64_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1638                         x86_64_xorps_reg_reg(cd, REG_FTMP2, d);
1639                         store_reg_to_var_flt(iptr->dst, d);
1640                         break;
1641
1642                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1643
1644                         var_to_reg_flt(s1, src, REG_FTMP1);
1645                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1646                         disp = dseg_adds8(cd, 0x8000000000000000);
1647                         M_FLTMOVE(s1, d);
1648                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1649                         x86_64_xorpd_reg_reg(cd, REG_FTMP2, d);
1650                         store_reg_to_var_flt(iptr->dst, d);
1651                         break;
1652
1653                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1654
1655                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1656                         var_to_reg_flt(s2, src, REG_FTMP2);
1657                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1658                         if (s1 == d) {
1659                                 x86_64_addss_reg_reg(cd, s2, d);
1660                         } else if (s2 == d) {
1661                                 x86_64_addss_reg_reg(cd, s1, d);
1662                         } else {
1663                                 M_FLTMOVE(s1, d);
1664                                 x86_64_addss_reg_reg(cd, s2, d);
1665                         }
1666                         store_reg_to_var_flt(iptr->dst, d);
1667                         break;
1668
1669                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1670
1671                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1672                         var_to_reg_flt(s2, src, REG_FTMP2);
1673                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1674                         if (s1 == d) {
1675                                 x86_64_addsd_reg_reg(cd, s2, d);
1676                         } else if (s2 == d) {
1677                                 x86_64_addsd_reg_reg(cd, s1, d);
1678                         } else {
1679                                 M_FLTMOVE(s1, d);
1680                                 x86_64_addsd_reg_reg(cd, s2, d);
1681                         }
1682                         store_reg_to_var_flt(iptr->dst, d);
1683                         break;
1684
1685                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1686
1687                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1688                         var_to_reg_flt(s2, src, REG_FTMP2);
1689                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1690                         if (s2 == d) {
1691                                 M_FLTMOVE(s2, REG_FTMP2);
1692                                 s2 = REG_FTMP2;
1693                         }
1694                         M_FLTMOVE(s1, d);
1695                         x86_64_subss_reg_reg(cd, s2, d);
1696                         store_reg_to_var_flt(iptr->dst, d);
1697                         break;
1698
1699                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1700
1701                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1702                         var_to_reg_flt(s2, src, REG_FTMP2);
1703                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1704                         if (s2 == d) {
1705                                 M_FLTMOVE(s2, REG_FTMP2);
1706                                 s2 = REG_FTMP2;
1707                         }
1708                         M_FLTMOVE(s1, d);
1709                         x86_64_subsd_reg_reg(cd, s2, d);
1710                         store_reg_to_var_flt(iptr->dst, d);
1711                         break;
1712
1713                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1714
1715                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1716                         var_to_reg_flt(s2, src, REG_FTMP2);
1717                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1718                         if (s1 == d) {
1719                                 x86_64_mulss_reg_reg(cd, s2, d);
1720                         } else if (s2 == d) {
1721                                 x86_64_mulss_reg_reg(cd, s1, d);
1722                         } else {
1723                                 M_FLTMOVE(s1, d);
1724                                 x86_64_mulss_reg_reg(cd, s2, d);
1725                         }
1726                         store_reg_to_var_flt(iptr->dst, d);
1727                         break;
1728
1729                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1730
1731                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1732                         var_to_reg_flt(s2, src, REG_FTMP2);
1733                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1734                         if (s1 == d) {
1735                                 x86_64_mulsd_reg_reg(cd, s2, d);
1736                         } else if (s2 == d) {
1737                                 x86_64_mulsd_reg_reg(cd, s1, d);
1738                         } else {
1739                                 M_FLTMOVE(s1, d);
1740                                 x86_64_mulsd_reg_reg(cd, s2, d);
1741                         }
1742                         store_reg_to_var_flt(iptr->dst, d);
1743                         break;
1744
1745                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1746
1747                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1748                         var_to_reg_flt(s2, src, REG_FTMP2);
1749                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1750                         if (s2 == d) {
1751                                 M_FLTMOVE(s2, REG_FTMP2);
1752                                 s2 = REG_FTMP2;
1753                         }
1754                         M_FLTMOVE(s1, d);
1755                         x86_64_divss_reg_reg(cd, s2, d);
1756                         store_reg_to_var_flt(iptr->dst, d);
1757                         break;
1758
1759                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1760
1761                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1762                         var_to_reg_flt(s2, src, REG_FTMP2);
1763                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1764                         if (s2 == d) {
1765                                 M_FLTMOVE(s2, REG_FTMP2);
1766                                 s2 = REG_FTMP2;
1767                         }
1768                         M_FLTMOVE(s1, d);
1769                         x86_64_divsd_reg_reg(cd, s2, d);
1770                         store_reg_to_var_flt(iptr->dst, d);
1771                         break;
1772
1773                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1774
1775                         var_to_reg_int(s1, src, REG_ITMP1);
1776                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1777                         x86_64_cvtsi2ss_reg_reg(cd, s1, d);
1778                         store_reg_to_var_flt(iptr->dst, d);
1779                         break;
1780
1781                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1782
1783                         var_to_reg_int(s1, src, REG_ITMP1);
1784                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1785                         x86_64_cvtsi2sd_reg_reg(cd, s1, d);
1786                         store_reg_to_var_flt(iptr->dst, d);
1787                         break;
1788
1789                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1790
1791                         var_to_reg_int(s1, src, REG_ITMP1);
1792                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1793                         x86_64_cvtsi2ssq_reg_reg(cd, s1, d);
1794                         store_reg_to_var_flt(iptr->dst, d);
1795                         break;
1796
1797                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1798
1799                         var_to_reg_int(s1, src, REG_ITMP1);
1800                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1801                         x86_64_cvtsi2sdq_reg_reg(cd, s1, d);
1802                         store_reg_to_var_flt(iptr->dst, d);
1803                         break;
1804
1805                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1806
1807                         var_to_reg_flt(s1, src, REG_FTMP1);
1808                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1809                         x86_64_cvttss2si_reg_reg(cd, s1, d);
1810                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1811                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1812                         x86_64_jcc(cd, X86_64_CC_NE, a);
1813                         M_FLTMOVE(s1, REG_FTMP1);
1814                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2i, REG_ITMP2);
1815                         x86_64_call_reg(cd, REG_ITMP2);
1816                         M_INTMOVE(REG_RESULT, d);
1817                         store_reg_to_var_int(iptr->dst, d);
1818                         break;
1819
1820                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1821
1822                         var_to_reg_flt(s1, src, REG_FTMP1);
1823                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1824                         x86_64_cvttsd2si_reg_reg(cd, s1, d);
1825                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1826                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1827                         x86_64_jcc(cd, X86_64_CC_NE, a);
1828                         M_FLTMOVE(s1, REG_FTMP1);
1829                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2i, REG_ITMP2);
1830                         x86_64_call_reg(cd, REG_ITMP2);
1831                         M_INTMOVE(REG_RESULT, d);
1832                         store_reg_to_var_int(iptr->dst, d);
1833                         break;
1834
1835                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1836
1837                         var_to_reg_flt(s1, src, REG_FTMP1);
1838                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1839                         x86_64_cvttss2siq_reg_reg(cd, s1, d);
1840                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1841                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1842                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1843                         x86_64_jcc(cd, X86_64_CC_NE, a);
1844                         M_FLTMOVE(s1, REG_FTMP1);
1845                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2l, REG_ITMP2);
1846                         x86_64_call_reg(cd, REG_ITMP2);
1847                         M_INTMOVE(REG_RESULT, d);
1848                         store_reg_to_var_int(iptr->dst, d);
1849                         break;
1850
1851                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1852
1853                         var_to_reg_flt(s1, src, REG_FTMP1);
1854                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1855                         x86_64_cvttsd2siq_reg_reg(cd, s1, d);
1856                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1857                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1858                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1859                         x86_64_jcc(cd, X86_64_CC_NE, a);
1860                         M_FLTMOVE(s1, REG_FTMP1);
1861                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2l, REG_ITMP2);
1862                         x86_64_call_reg(cd, REG_ITMP2);
1863                         M_INTMOVE(REG_RESULT, d);
1864                         store_reg_to_var_int(iptr->dst, d);
1865                         break;
1866
1867                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1868
1869                         var_to_reg_flt(s1, src, REG_FTMP1);
1870                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1871                         x86_64_cvtss2sd_reg_reg(cd, s1, d);
1872                         store_reg_to_var_flt(iptr->dst, d);
1873                         break;
1874
1875                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1876
1877                         var_to_reg_flt(s1, src, REG_FTMP1);
1878                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1879                         x86_64_cvtsd2ss_reg_reg(cd, s1, d);
1880                         store_reg_to_var_flt(iptr->dst, d);
1881                         break;
1882
1883                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1884                                           /* == => 0, < => 1, > => -1 */
1885
1886                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1887                         var_to_reg_flt(s2, src, REG_FTMP2);
1888                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1889                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1890                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1891                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1892                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1893                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1894                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1895                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1896                         store_reg_to_var_int(iptr->dst, d);
1897                         break;
1898
1899                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1900                                           /* == => 0, < => 1, > => -1 */
1901
1902                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1903                         var_to_reg_flt(s2, src, REG_FTMP2);
1904                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1905                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1906                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1907                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1908                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1909                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1910                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1911                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1912                         store_reg_to_var_int(iptr->dst, d);
1913                         break;
1914
1915                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1916                                           /* == => 0, < => 1, > => -1 */
1917
1918                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1919                         var_to_reg_flt(s2, src, REG_FTMP2);
1920                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1921                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1922                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1923                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1924                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1925                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1926                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1927                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1928                         store_reg_to_var_int(iptr->dst, d);
1929                         break;
1930
1931                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1932                                           /* == => 0, < => 1, > => -1 */
1933
1934                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1935                         var_to_reg_flt(s2, src, REG_FTMP2);
1936                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1937                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1938                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1939                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1940                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1941                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1942                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1943                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1944                         store_reg_to_var_int(iptr->dst, d);
1945                         break;
1946
1947
1948                 /* memory operations **************************************************/
1949
1950                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1951
1952                         var_to_reg_int(s1, src, REG_ITMP1);
1953                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1954                         gen_nullptr_check(s1);
1955                         x86_64_movl_membase_reg(cd, s1, OFFSET(java_arrayheader, size), d);
1956                         store_reg_to_var_int(iptr->dst, d);
1957                         break;
1958
1959                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1960
1961                         var_to_reg_int(s1, src->prev, REG_ITMP1);
1962                         var_to_reg_int(s2, src, REG_ITMP2);
1963                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1964                         if (iptr->op1 == 0) {
1965                                 gen_nullptr_check(s1);
1966                                 gen_bound_check;
1967                         }
1968                         x86_64_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1969                         store_reg_to_var_int(iptr->dst, d);
1970                         break;
1971
1972                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1973
1974                         var_to_reg_int(s1, src->prev, REG_ITMP1);
1975                         var_to_reg_int(s2, src, REG_ITMP2);
1976                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1977                         if (iptr->op1 == 0) {
1978                                 gen_nullptr_check(s1);
1979                                 gen_bound_check;
1980                         }
1981                         x86_64_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1982                         store_reg_to_var_int(iptr->dst, d);
1983                         break;
1984
1985                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1986
1987                         var_to_reg_int(s1, src->prev, REG_ITMP1);
1988                         var_to_reg_int(s2, src, REG_ITMP2);
1989                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1990                         if (iptr->op1 == 0) {
1991                                 gen_nullptr_check(s1);
1992                                 gen_bound_check;
1993                         }
1994                         x86_64_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1995                         store_reg_to_var_int(iptr->dst, d);
1996                         break;
1997
1998                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1999
2000                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2001                         var_to_reg_int(s2, src, REG_ITMP2);
2002                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2003                         if (iptr->op1 == 0) {
2004                                 gen_nullptr_check(s1);
2005                                 gen_bound_check;
2006                         }
2007                         x86_64_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2008                         store_reg_to_var_flt(iptr->dst, d);
2009                         break;
2010
2011                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2012
2013                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2014                         var_to_reg_int(s2, src, REG_ITMP2);
2015                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2016                         if (iptr->op1 == 0) {
2017                                 gen_nullptr_check(s1);
2018                                 gen_bound_check;
2019                         }
2020                         x86_64_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2021                         store_reg_to_var_flt(iptr->dst, d);
2022                         break;
2023
2024                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2025
2026                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2027                         var_to_reg_int(s2, src, REG_ITMP2);
2028                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2029                         if (iptr->op1 == 0) {
2030                                 gen_nullptr_check(s1);
2031                                 gen_bound_check;
2032                         }
2033                         x86_64_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2034                         store_reg_to_var_int(iptr->dst, d);
2035                         break;
2036
2037                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2038
2039                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2040                         var_to_reg_int(s2, src, REG_ITMP2);
2041                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2042                         if (iptr->op1 == 0) {
2043                                 gen_nullptr_check(s1);
2044                                 gen_bound_check;
2045                         }
2046                         x86_64_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2047                         store_reg_to_var_int(iptr->dst, d);
2048                         break;
2049
2050                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2051
2052                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2053                         var_to_reg_int(s2, src, REG_ITMP2);
2054                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2055                         if (iptr->op1 == 0) {
2056                                 gen_nullptr_check(s1);
2057                                 gen_bound_check;
2058                         }
2059                         x86_64_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2060                         store_reg_to_var_int(iptr->dst, d);
2061                         break;
2062
2063
2064                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2065
2066                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2067                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2068                         if (iptr->op1 == 0) {
2069                                 gen_nullptr_check(s1);
2070                                 gen_bound_check;
2071                         }
2072                         var_to_reg_int(s3, src, REG_ITMP3);
2073                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2074                         break;
2075
2076                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2077
2078                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2079                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2080                         if (iptr->op1 == 0) {
2081                                 gen_nullptr_check(s1);
2082                                 gen_bound_check;
2083                         }
2084                         var_to_reg_int(s3, src, REG_ITMP3);
2085                         x86_64_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2086                         break;
2087
2088                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2089
2090                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2091                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2092                         if (iptr->op1 == 0) {
2093                                 gen_nullptr_check(s1);
2094                                 gen_bound_check;
2095                         }
2096                         var_to_reg_flt(s3, src, REG_FTMP3);
2097                         x86_64_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2098                         break;
2099
2100                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2101
2102                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2103                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2104                         if (iptr->op1 == 0) {
2105                                 gen_nullptr_check(s1);
2106                                 gen_bound_check;
2107                         }
2108                         var_to_reg_flt(s3, src, REG_FTMP3);
2109                         x86_64_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2110                         break;
2111
2112                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2113
2114                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2115                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2116                         if (iptr->op1 == 0) {
2117                                 gen_nullptr_check(s1);
2118                                 gen_bound_check;
2119                         }
2120                         var_to_reg_int(s3, src, REG_ITMP3);
2121                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2122                         break;
2123
2124                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2125
2126                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2127                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2128                         if (iptr->op1 == 0) {
2129                                 gen_nullptr_check(s1);
2130                                 gen_bound_check;
2131                         }
2132                         var_to_reg_int(s3, src, REG_ITMP3);
2133                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2134                         break;
2135
2136                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2137
2138                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2139                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2140                         if (iptr->op1 == 0) {
2141                                 gen_nullptr_check(s1);
2142                                 gen_bound_check;
2143                         }
2144                         var_to_reg_int(s3, src, REG_ITMP3);
2145                         x86_64_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2146                         break;
2147
2148                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2149
2150                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2151                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2152 /*                      if (iptr->op1 == 0) { */
2153                                 gen_nullptr_check(s1);
2154                                 gen_bound_check;
2155 /*                      } */
2156                         var_to_reg_int(s3, src, REG_ITMP3);
2157
2158                         M_MOV(s1, rd->argintregs[0]);
2159                         M_MOV(s3, rd->argintregs[1]);
2160                         bte = iptr->val.a;
2161                         x86_64_mov_imm_reg(cd, (ptrint) bte->fp, REG_ITMP1);
2162                         x86_64_call_reg(cd, REG_ITMP1);
2163                         M_TEST(REG_RESULT);
2164                         M_BEQ(0);
2165                         codegen_addxstorerefs(cd, cd->mcodeptr);
2166
2167                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2168                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2169                         var_to_reg_int(s3, src, REG_ITMP3);
2170                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2171                         break;
2172
2173
2174                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2175
2176                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2177                         var_to_reg_int(s2, src, REG_ITMP2);
2178                         if (iptr->op1 == 0) {
2179                                 gen_nullptr_check(s1);
2180                                 gen_bound_check;
2181                         }
2182                         x86_64_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2183                         break;
2184
2185                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2186
2187                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2188                         var_to_reg_int(s2, src, REG_ITMP2);
2189                         if (iptr->op1 == 0) {
2190                                 gen_nullptr_check(s1);
2191                                 gen_bound_check;
2192                         }
2193
2194                         if (IS_IMM32(iptr->val.l)) {
2195                                 x86_64_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2196
2197                         } else {
2198                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2199                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2200                         }
2201                         break;
2202
2203                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2204
2205                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2206                         var_to_reg_int(s2, src, REG_ITMP2);
2207                         if (iptr->op1 == 0) {
2208                                 gen_nullptr_check(s1);
2209                                 gen_bound_check;
2210                         }
2211                         x86_64_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2212                         break;
2213
2214                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2215
2216                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2217                         var_to_reg_int(s2, src, REG_ITMP2);
2218                         if (iptr->op1 == 0) {
2219                                 gen_nullptr_check(s1);
2220                                 gen_bound_check;
2221                         }
2222                         x86_64_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2223                         break;
2224
2225                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2226
2227                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2228                         var_to_reg_int(s2, src, REG_ITMP2);
2229                         if (iptr->op1 == 0) {
2230                                 gen_nullptr_check(s1);
2231                                 gen_bound_check;
2232                         }
2233                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2234                         break;
2235
2236                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2237
2238                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2239                         var_to_reg_int(s2, src, REG_ITMP2);
2240                         if (iptr->op1 == 0) {
2241                                 gen_nullptr_check(s1);
2242                                 gen_bound_check;
2243                         }
2244                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2245                         break;
2246
2247
2248                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2249                                       /* op1 = type, val.a = field address            */
2250
2251                         if (!iptr->val.a) {
2252                                 disp = dseg_addaddress(cd, NULL);
2253
2254                                 codegen_addpatchref(cd, cd->mcodeptr,
2255                                                                         PATCHER_get_putstatic,
2256                                                                         (unresolved_field *) iptr->target, disp);
2257
2258                                 if (opt_showdisassemble) {
2259                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2260                                 }
2261
2262                         } else {
2263                                 fieldinfo *fi = iptr->val.a;
2264
2265                                 disp = dseg_addaddress(cd, &(fi->value));
2266
2267                                 if (!fi->class->initialized) {
2268                                         codegen_addpatchref(cd, cd->mcodeptr,
2269                                                                                 PATCHER_clinit, fi->class, 0);
2270
2271                                         if (opt_showdisassemble) {
2272                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2273                                         }
2274                                 }
2275                         }
2276
2277                         /* This approach is much faster than moving the field address     */
2278                         /* inline into a register. */
2279                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP2);
2280                         switch (iptr->op1) {
2281                         case TYPE_INT:
2282                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2283                                 x86_64_movl_membase_reg(cd, REG_ITMP2, 0, d);
2284                                 store_reg_to_var_int(iptr->dst, d);
2285                                 break;
2286                         case TYPE_LNG:
2287                         case TYPE_ADR:
2288                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2289                                 x86_64_mov_membase_reg(cd, REG_ITMP2, 0, d);
2290                                 store_reg_to_var_int(iptr->dst, d);
2291                                 break;
2292                         case TYPE_FLT:
2293                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2294                                 x86_64_movss_membase_reg(cd, REG_ITMP2, 0, d);
2295                                 store_reg_to_var_flt(iptr->dst, d);
2296                                 break;
2297                         case TYPE_DBL:
2298                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2299                                 x86_64_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2300                                 store_reg_to_var_flt(iptr->dst, d);
2301                                 break;
2302                         }
2303                         break;
2304
2305                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2306                                       /* op1 = type, val.a = field address            */
2307
2308                         if (!iptr->val.a) {
2309                                 disp = dseg_addaddress(cd, NULL);
2310
2311                                 codegen_addpatchref(cd, cd->mcodeptr,
2312                                                                         PATCHER_get_putstatic,
2313                                                                         (unresolved_field *) iptr->target, disp);
2314
2315                                 if (opt_showdisassemble) {
2316                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2317                                 }
2318
2319                         } else {
2320                                 fieldinfo *fi = iptr->val.a;
2321
2322                                 disp = dseg_addaddress(cd, &(fi->value));
2323
2324                                 if (!fi->class->initialized) {
2325                                         codegen_addpatchref(cd, cd->mcodeptr,
2326                                                                                 PATCHER_clinit, fi->class, 0);
2327
2328                                         if (opt_showdisassemble) {
2329                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2330                                         }
2331                                 }
2332                         }
2333
2334                         /* This approach is much faster than moving the field address     */
2335                         /* inline into a register. */
2336                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP2);
2337                         switch (iptr->op1) {
2338                         case TYPE_INT:
2339                                 var_to_reg_int(s2, src, REG_ITMP1);
2340                                 x86_64_movl_reg_membase(cd, s2, REG_ITMP2, 0);
2341                                 break;
2342                         case TYPE_LNG:
2343                         case TYPE_ADR:
2344                                 var_to_reg_int(s2, src, REG_ITMP1);
2345                                 x86_64_mov_reg_membase(cd, s2, REG_ITMP2, 0);
2346                                 break;
2347                         case TYPE_FLT:
2348                                 var_to_reg_flt(s2, src, REG_FTMP1);
2349                                 x86_64_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2350                                 break;
2351                         case TYPE_DBL:
2352                                 var_to_reg_flt(s2, src, REG_FTMP1);
2353                                 x86_64_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2354                                 break;
2355                         }
2356                         break;
2357
2358                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2359                                           /* val = value (in current instruction)     */
2360                                           /* op1 = type, val.a = field address (in    */
2361                                           /* following NOP)                           */
2362
2363                         if (!iptr[1].val.a) {
2364                                 disp = dseg_addaddress(cd, NULL);
2365
2366                                 codegen_addpatchref(cd, cd->mcodeptr,
2367                                                                         PATCHER_get_putstatic,
2368                                                                         (unresolved_field *) iptr[1].target, disp);
2369
2370                                 if (opt_showdisassemble) {
2371                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2372                                 }
2373
2374                         } else {
2375                                 fieldinfo *fi = iptr[1].val.a;
2376
2377                                 disp = dseg_addaddress(cd, &(fi->value));
2378
2379                                 if (!fi->class->initialized) {
2380                                         codegen_addpatchref(cd, cd->mcodeptr,
2381                                                                                 PATCHER_clinit, fi->class, 0);
2382
2383                                         if (opt_showdisassemble) {
2384                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2385                                         }
2386                                 }
2387                         }
2388
2389                         /* This approach is much faster than moving the field address     */
2390                         /* inline into a register. */
2391                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP1);
2392                         switch (iptr->op1) {
2393                         case TYPE_INT:
2394                         case TYPE_FLT:
2395                                 x86_64_movl_imm_membase(cd, iptr->val.i, REG_ITMP1, 0);
2396                                 break;
2397                         case TYPE_LNG:
2398                         case TYPE_ADR:
2399                         case TYPE_DBL:
2400                                 if (IS_IMM32(iptr->val.l)) {
2401                                         x86_64_mov_imm_membase(cd, iptr->val.l, REG_ITMP1, 0);
2402                                 } else {
2403                                         x86_64_movl_imm_membase(cd, iptr->val.l, REG_ITMP1, 0);
2404                                         x86_64_movl_imm_membase(cd, iptr->val.l >> 32, REG_ITMP1, 4);
2405                                 }
2406                                 break;
2407                         }
2408                         break;
2409
2410                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2411                                       /* op1 = type, val.i = field offset             */
2412
2413                         var_to_reg_int(s1, src, REG_ITMP1);
2414                         gen_nullptr_check(s1);
2415
2416                         if (!iptr->val.a) {
2417                                 codegen_addpatchref(cd, cd->mcodeptr,
2418                                                                         PATCHER_get_putfield,
2419                                                                         (unresolved_field *) iptr->target, 0);
2420
2421                                 if (opt_showdisassemble) {
2422                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2423                                 }
2424
2425                                 a = 0;
2426
2427                         } else {
2428                                 a = ((fieldinfo *) (iptr->val.a))->offset;
2429                         }
2430
2431                         switch (iptr->op1) {
2432                         case TYPE_INT:
2433                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2434                                 x86_64_movl_membase32_reg(cd, s1, a, d);
2435                                 store_reg_to_var_int(iptr->dst, d);
2436                                 break;
2437                         case TYPE_LNG:
2438                         case TYPE_ADR:
2439                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2440                                 x86_64_mov_membase32_reg(cd, s1, a, d);
2441                                 store_reg_to_var_int(iptr->dst, d);
2442                                 break;
2443                         case TYPE_FLT:
2444                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2445                                 x86_64_movss_membase32_reg(cd, s1, a, d);
2446                                 store_reg_to_var_flt(iptr->dst, d);
2447                                 break;
2448                         case TYPE_DBL:
2449                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2450                                 x86_64_movsd_membase32_reg(cd, s1, a, d);
2451                                 store_reg_to_var_flt(iptr->dst, d);
2452                                 break;
2453                         }
2454                         break;
2455
2456                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2457                                       /* op1 = type, val.i = field offset             */
2458
2459                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2460                         gen_nullptr_check(s1);
2461                         if (IS_INT_LNG_TYPE(iptr->op1)) {
2462                                 var_to_reg_int(s2, src, REG_ITMP2);
2463                         } else {
2464                                 var_to_reg_flt(s2, src, REG_FTMP2);
2465                         }
2466
2467                         if (!iptr->val.a) {
2468                                 codegen_addpatchref(cd, cd->mcodeptr,
2469                                                                         PATCHER_get_putfield,
2470                                                                         (unresolved_field *) iptr->target, 0);
2471
2472                                 if (opt_showdisassemble) {
2473                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2474                                 }
2475
2476                                 a = 0;
2477
2478                         } else {
2479                                 a = ((fieldinfo *) (iptr->val.a))->offset;
2480                         }
2481
2482                         switch (iptr->op1) {
2483                         case TYPE_INT:
2484                                 x86_64_movl_reg_membase32(cd, s2, s1, a);
2485                                 break;
2486                         case TYPE_LNG:
2487                         case TYPE_ADR:
2488                                 x86_64_mov_reg_membase32(cd, s2, s1, a);
2489                                 break;
2490                         case TYPE_FLT:
2491                                 x86_64_movss_reg_membase32(cd, s2, s1, a);
2492                                 break;
2493                         case TYPE_DBL:
2494                                 x86_64_movsd_reg_membase32(cd, s2, s1, a);
2495                                 break;
2496                         }
2497                         break;
2498
2499                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2500                                           /* val = value (in current instruction)     */
2501                                           /* op1 = type, val.a = field address (in    */
2502                                           /* following NOP)                           */
2503
2504                         var_to_reg_int(s1, src, REG_ITMP1);
2505                         gen_nullptr_check(s1);
2506
2507                         if (!iptr[1].val.a) {
2508                                 codegen_addpatchref(cd, cd->mcodeptr,
2509                                                                         PATCHER_putfieldconst,
2510                                                                         (unresolved_field *) iptr[1].target, 0);
2511
2512                                 if (opt_showdisassemble) {
2513                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2514                                 }
2515
2516                                 a = 0;
2517
2518                         } else {
2519                                 a = ((fieldinfo *) (iptr[1].val.a))->offset;
2520                         }
2521
2522                         switch (iptr->op1) {
2523                         case TYPE_INT:
2524                         case TYPE_FLT:
2525                                 x86_64_movl_imm_membase32(cd, iptr->val.i, s1, a);
2526                                 break;
2527                         case TYPE_LNG:
2528                         case TYPE_ADR:
2529                         case TYPE_DBL:
2530                                 /* We can only optimize the move, if the class is resolved.   */
2531                                 /* Otherwise we don't know what to patch.                     */
2532                                 if (iptr[1].val.a && IS_IMM32(iptr->val.l)) {
2533                                         x86_64_mov_imm_membase32(cd, iptr->val.l, s1, a);
2534                                 } else {
2535                                         x86_64_movl_imm_membase32(cd, iptr->val.l, s1, a);
2536                                         x86_64_movl_imm_membase32(cd, iptr->val.l >> 32, s1, a + 4);
2537                                 }
2538                                 break;
2539                         }
2540                         break;
2541
2542
2543                 /* branch operations **************************************************/
2544
2545                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2546
2547                         var_to_reg_int(s1, src, REG_ITMP1);
2548                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2549
2550                         x86_64_call_imm(cd, 0); /* passing exception pointer              */
2551                         x86_64_pop_reg(cd, REG_ITMP2_XPC);
2552
2553                         x86_64_mov_imm_reg(cd, (ptrint) asm_handle_exception, REG_ITMP3);
2554                         x86_64_jmp_reg(cd, REG_ITMP3);
2555                         break;
2556
2557                 case ICMD_GOTO:         /* ... ==> ...                                */
2558                                         /* op1 = target JavaVM pc                     */
2559
2560                         x86_64_jmp_imm(cd, 0);
2561                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2562                         break;
2563
2564                 case ICMD_JSR:          /* ... ==> ...                                */
2565                                         /* op1 = target JavaVM pc                     */
2566
2567                         x86_64_call_imm(cd, 0);
2568                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2569                         break;
2570
2571                 case ICMD_RET:          /* ... ==> ...                                */
2572                                         /* op1 = local variable                       */
2573
2574                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2575                         var_to_reg_int(s1, var, REG_ITMP1);
2576                         x86_64_jmp_reg(cd, s1);
2577                         break;
2578
2579                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2580                                         /* op1 = target JavaVM pc                     */
2581
2582                         if (src->flags & INMEMORY) {
2583                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2584
2585                         } else {
2586                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2587                         }
2588                         x86_64_jcc(cd, X86_64_CC_E, 0);
2589                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2590                         break;
2591
2592                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2593                                         /* op1 = target JavaVM pc                     */
2594
2595                         if (src->flags & INMEMORY) {
2596                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2597
2598                         } else {
2599                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2600                         }
2601                         x86_64_jcc(cd, X86_64_CC_NE, 0);
2602                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2603                         break;
2604
2605                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2606                                         /* op1 = target JavaVM pc, val.i = constant   */
2607
2608                         x86_64_emit_ifcc(cd, X86_64_CC_E, src, iptr);
2609                         break;
2610
2611                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2612                                         /* op1 = target JavaVM pc, val.i = constant   */
2613
2614                         x86_64_emit_ifcc(cd, X86_64_CC_L, src, iptr);
2615                         break;
2616
2617                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2618                                         /* op1 = target JavaVM pc, val.i = constant   */
2619
2620                         x86_64_emit_ifcc(cd, X86_64_CC_LE, src, iptr);
2621                         break;
2622
2623                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2624                                         /* op1 = target JavaVM pc, val.i = constant   */
2625
2626                         x86_64_emit_ifcc(cd, X86_64_CC_NE, src, iptr);
2627                         break;
2628
2629                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2630                                         /* op1 = target JavaVM pc, val.i = constant   */
2631
2632                         x86_64_emit_ifcc(cd, X86_64_CC_G, src, iptr);
2633                         break;
2634
2635                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2636                                         /* op1 = target JavaVM pc, val.i = constant   */
2637
2638                         x86_64_emit_ifcc(cd, X86_64_CC_GE, src, iptr);
2639                         break;
2640
2641                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2642                                         /* op1 = target JavaVM pc, val.l = constant   */
2643
2644                         x86_64_emit_if_lcc(cd, X86_64_CC_E, src, iptr);
2645                         break;
2646
2647                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2648                                         /* op1 = target JavaVM pc, val.l = constant   */
2649
2650                         x86_64_emit_if_lcc(cd, X86_64_CC_L, src, iptr);
2651                         break;
2652
2653                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2654                                         /* op1 = target JavaVM pc, val.l = constant   */
2655
2656                         x86_64_emit_if_lcc(cd, X86_64_CC_LE, src, iptr);
2657                         break;
2658
2659                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2660                                         /* op1 = target JavaVM pc, val.l = constant   */
2661
2662                         x86_64_emit_if_lcc(cd, X86_64_CC_NE, src, iptr);
2663                         break;
2664
2665                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2666                                         /* op1 = target JavaVM pc, val.l = constant   */
2667
2668                         x86_64_emit_if_lcc(cd, X86_64_CC_G, src, iptr);
2669                         break;
2670
2671                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2672                                         /* op1 = target JavaVM pc, val.l = constant   */
2673
2674                         x86_64_emit_if_lcc(cd, X86_64_CC_GE, src, iptr);
2675                         break;
2676
2677                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2678                                         /* op1 = target JavaVM pc                     */
2679
2680                         x86_64_emit_if_icmpcc(cd, X86_64_CC_E, src, iptr);
2681                         break;
2682
2683                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2684                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2685
2686                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_E, src, iptr);
2687                         break;
2688
2689                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2690                                         /* op1 = target JavaVM pc                     */
2691
2692                         x86_64_emit_if_icmpcc(cd, X86_64_CC_NE, src, iptr);
2693                         break;
2694
2695                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2696                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2697
2698                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_NE, src, iptr);
2699                         break;
2700
2701                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2702                                         /* op1 = target JavaVM pc                     */
2703
2704                         x86_64_emit_if_icmpcc(cd, X86_64_CC_L, src, iptr);
2705                         break;
2706
2707                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2708                                     /* op1 = target JavaVM pc                     */
2709
2710                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_L, src, iptr);
2711                         break;
2712
2713                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2714                                         /* op1 = target JavaVM pc                     */
2715
2716                         x86_64_emit_if_icmpcc(cd, X86_64_CC_G, src, iptr);
2717                         break;
2718
2719                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2720                                 /* op1 = target JavaVM pc                     */
2721
2722                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_G, src, iptr);
2723                         break;
2724
2725                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2726                                         /* op1 = target JavaVM pc                     */
2727
2728                         x86_64_emit_if_icmpcc(cd, X86_64_CC_LE, src, iptr);
2729                         break;
2730
2731                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2732                                         /* op1 = target JavaVM pc                     */
2733
2734                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_LE, src, iptr);
2735                         break;
2736
2737                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2738                                         /* op1 = target JavaVM pc                     */
2739
2740                         x86_64_emit_if_icmpcc(cd, X86_64_CC_GE, src, iptr);
2741                         break;
2742
2743                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2744                                     /* op1 = target JavaVM pc                     */
2745
2746                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_GE, src, iptr);
2747                         break;
2748
2749                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2750
2751                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2752                         break;
2753
2754                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2755                                         /* val.i = constant                           */
2756
2757                         var_to_reg_int(s1, src, REG_ITMP1);
2758                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2759                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2760                                 if (s1 == d) {
2761                                         M_INTMOVE(s1, REG_ITMP1);
2762                                         s1 = REG_ITMP1;
2763                                 }
2764                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2765                         }
2766                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2767                         x86_64_testl_reg_reg(cd, s1, s1);
2768                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_E, REG_ITMP2, d);
2769                         store_reg_to_var_int(iptr->dst, d);
2770                         break;
2771
2772                 case ICMD_IFNE_ICONST:  /* ..., value ==> ..., constant               */
2773                                         /* val.i = constant                           */
2774
2775                         var_to_reg_int(s1, src, REG_ITMP1);
2776                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2777                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2778                                 if (s1 == d) {
2779                                         M_INTMOVE(s1, REG_ITMP1);
2780                                         s1 = REG_ITMP1;
2781                                 }
2782                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2783                         }
2784                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2785                         x86_64_testl_reg_reg(cd, s1, s1);
2786                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_NE, REG_ITMP2, d);
2787                         store_reg_to_var_int(iptr->dst, d);
2788                         break;
2789
2790                 case ICMD_IFLT_ICONST:  /* ..., value ==> ..., constant               */
2791                                         /* val.i = constant                           */
2792
2793                         var_to_reg_int(s1, src, REG_ITMP1);
2794                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2795                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2796                                 if (s1 == d) {
2797                                         M_INTMOVE(s1, REG_ITMP1);
2798                                         s1 = REG_ITMP1;
2799                                 }
2800                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2801                         }
2802                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2803                         x86_64_testl_reg_reg(cd, s1, s1);
2804                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_L, REG_ITMP2, d);
2805                         store_reg_to_var_int(iptr->dst, d);
2806                         break;
2807
2808                 case ICMD_IFGE_ICONST:  /* ..., value ==> ..., constant               */
2809                                         /* val.i = constant                           */
2810
2811                         var_to_reg_int(s1, src, REG_ITMP1);
2812                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2813                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2814                                 if (s1 == d) {
2815                                         M_INTMOVE(s1, REG_ITMP1);
2816                                         s1 = REG_ITMP1;
2817                                 }
2818                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2819                         }
2820                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2821                         x86_64_testl_reg_reg(cd, s1, s1);
2822                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_GE, REG_ITMP2, d);
2823                         store_reg_to_var_int(iptr->dst, d);
2824                         break;
2825
2826                 case ICMD_IFGT_ICONST:  /* ..., value ==> ..., constant               */
2827                                         /* val.i = constant                           */
2828
2829                         var_to_reg_int(s1, src, REG_ITMP1);
2830                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2831                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2832                                 if (s1 == d) {
2833                                         M_INTMOVE(s1, REG_ITMP1);
2834                                         s1 = REG_ITMP1;
2835                                 }
2836                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2837                         }
2838                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2839                         x86_64_testl_reg_reg(cd, s1, s1);
2840                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP2, d);
2841                         store_reg_to_var_int(iptr->dst, d);
2842                         break;
2843
2844                 case ICMD_IFLE_ICONST:  /* ..., value ==> ..., constant               */
2845                                         /* val.i = constant                           */
2846
2847                         var_to_reg_int(s1, src, REG_ITMP1);
2848                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2849                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2850                                 if (s1 == d) {
2851                                         M_INTMOVE(s1, REG_ITMP1);
2852                                         s1 = REG_ITMP1;
2853                                 }
2854                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2855                         }
2856                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2857                         x86_64_testl_reg_reg(cd, s1, s1);
2858                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, d);
2859                         store_reg_to_var_int(iptr->dst, d);
2860                         break;
2861
2862
2863                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2864                 case ICMD_LRETURN:
2865                 case ICMD_ARETURN:
2866
2867                         var_to_reg_int(s1, src, REG_RESULT);
2868                         M_INTMOVE(s1, REG_RESULT);
2869
2870                         goto nowperformreturn;
2871
2872                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2873                 case ICMD_DRETURN:
2874
2875                         var_to_reg_flt(s1, src, REG_FRESULT);
2876                         M_FLTMOVE(s1, REG_FRESULT);
2877
2878                         goto nowperformreturn;
2879
2880                 case ICMD_RETURN:      /* ...  ==> ...                                */
2881
2882 nowperformreturn:
2883                         {
2884                         s4 i, p;
2885
2886                         p = parentargs_base;
2887
2888                         /* call trace function */
2889                         if (runverbose) {
2890                                 x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
2891
2892                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2893                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2894
2895                                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
2896                                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2897                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2898                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2899
2900                                 x86_64_mov_imm_reg(cd, (u8) builtin_displaymethodstop, REG_ITMP1);
2901                                 x86_64_call_reg(cd, REG_ITMP1);
2902
2903                                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2904                                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2905
2906                                 x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
2907                         }
2908
2909 #if defined(USE_THREADS)
2910                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2911                                 M_ALD(rd->argintregs[0], REG_SP, rd->memuse * 8);
2912
2913                                 /* we need to save the proper return value */
2914                                 switch (iptr->opc) {
2915                                 case ICMD_IRETURN:
2916                                 case ICMD_ARETURN:
2917                                 case ICMD_LRETURN:
2918                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2919                                         break;
2920                                 case ICMD_FRETURN:
2921                                 case ICMD_DRETURN:
2922                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2923                                         break;
2924                                 }
2925
2926                                 x86_64_mov_imm_reg(cd, (ptrint) builtin_monitorexit, REG_ITMP1);
2927                                 x86_64_call_reg(cd, REG_ITMP1);
2928
2929                                 /* and now restore the proper return value */
2930                                 switch (iptr->opc) {
2931                                 case ICMD_IRETURN:
2932                                 case ICMD_ARETURN:
2933                                 case ICMD_LRETURN:
2934                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2935                                         break;
2936                                 case ICMD_FRETURN:
2937                                 case ICMD_DRETURN:
2938                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2939                                         break;
2940                                 }
2941                         }
2942 #endif
2943
2944                         /* restore saved registers */
2945
2946                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2947                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2948                         }
2949                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2950                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2951                         }
2952
2953                         /* deallocate stack */
2954
2955                         if (parentargs_base)
2956                                 M_AADD_IMM(parentargs_base * 8, REG_SP);
2957
2958                         M_RET;
2959                         }
2960                         break;
2961
2962
2963                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2964                         {
2965                                 s4 i, l, *s4ptr;
2966                                 void **tptr;
2967
2968                                 tptr = (void **) iptr->target;
2969
2970                                 s4ptr = iptr->val.a;
2971                                 l = s4ptr[1];                          /* low     */
2972                                 i = s4ptr[2];                          /* high    */
2973
2974                                 var_to_reg_int(s1, src, REG_ITMP1);
2975                                 M_INTMOVE(s1, REG_ITMP1);
2976                                 if (l != 0) {
2977                                         x86_64_alul_imm_reg(cd, X86_64_SUB, l, REG_ITMP1);
2978                                 }
2979                                 i = i - l + 1;
2980
2981                 /* range check */
2982                                 x86_64_alul_imm_reg(cd, X86_64_CMP, i - 1, REG_ITMP1);
2983                                 x86_64_jcc(cd, X86_64_CC_A, 0);
2984
2985                 /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[0]), cd->mcodeptr); */
2986                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
2987
2988                                 /* build jump table top down and use address of lowest entry */
2989
2990                 /* s4ptr += 3 + i; */
2991                                 tptr += i;
2992
2993                                 while (--i >= 0) {
2994                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
2995                                         --tptr;
2996                                 }
2997
2998                                 /* length of dataseg after last dseg_addtarget is used by load */
2999
3000                                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2);
3001                                 dseg_adddata(cd, cd->mcodeptr);
3002                                 x86_64_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
3003                                 x86_64_jmp_reg(cd, REG_ITMP1);
3004                         }
3005                         break;
3006
3007
3008                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
3009                         {
3010                                 s4 i, l, val, *s4ptr;
3011                                 void **tptr;
3012
3013                                 tptr = (void **) iptr->target;
3014
3015                                 s4ptr = iptr->val.a;
3016                                 l = s4ptr[0];                          /* default  */
3017                                 i = s4ptr[1];                          /* count    */
3018
3019                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
3020                                 var_to_reg_int(s1, src, REG_ITMP1);    /* reg compare should always be faster */
3021                                 while (--i >= 0) {
3022                                         s4ptr += 2;
3023                                         ++tptr;
3024
3025                                         val = s4ptr[0];
3026                                         x86_64_alul_imm_reg(cd, X86_64_CMP, val, s1);
3027                                         x86_64_jcc(cd, X86_64_CC_E, 0);
3028                                         codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3029                                 }
3030
3031                                 x86_64_jmp_imm(cd, 0);
3032
3033                                 tptr = (void **) iptr->target;
3034                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3035                         }
3036                         break;
3037
3038
3039                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
3040                                         /* op1 = arg count val.a = builtintable entry */
3041
3042                         bte = iptr->val.a;
3043                         md = bte->md;
3044                         goto gen_method;
3045
3046                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
3047                                         /* op1 = arg count, val.a = method pointer    */
3048
3049                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
3050                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
3051                 case ICMD_INVOKEINTERFACE:
3052
3053                         lm = iptr->val.a;
3054
3055                         if (lm)
3056                                 md = lm->parseddesc;
3057                         else {
3058                                 unresolved_method *um = iptr->target;
3059                                 md = um->methodref->parseddesc.md;
3060                         }
3061
3062 gen_method:
3063                         s3 = iptr->op1;
3064
3065                         MCODECHECK((20 * s3) + 128);
3066
3067                         /* copy arguments to registers or stack location */
3068
3069                         for (s3 = s3 - 1; s3 >= 0; s3--, src = src->prev) {
3070                                 if (src->varkind == ARGVAR)
3071                                         continue;
3072                                 if (IS_INT_LNG_TYPE(src->type)) {
3073                                         if (!md->params[s3].inmemory) {
3074                                                 s1 = rd->argintregs[md->params[s3].regoff];
3075                                                 var_to_reg_int(d, src, s1);
3076                                                 M_INTMOVE(d, s1);
3077                                         } else {
3078                                                 var_to_reg_int(d, src, REG_ITMP1);
3079                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
3080                                         }
3081
3082                                 } else {
3083                                         if (!md->params[s3].inmemory) {
3084                                                 s1 = rd->argfltregs[md->params[s3].regoff];
3085                                                 var_to_reg_flt(d, src, s1);
3086                                                 M_FLTMOVE(d, s1);
3087                                         } else {
3088                                                 var_to_reg_flt(d, src, REG_FTMP1);
3089                                                 M_DST(d, REG_SP, md->params[s3].regoff * 8);
3090                                         }
3091                                 }
3092                         }
3093
3094                         switch (iptr->opc) {
3095                         case ICMD_BUILTIN:
3096                                 if (iptr->target) {
3097                                         codegen_addpatchref(cd, cd->mcodeptr,
3098                                                                                 bte->fp, iptr->target, 0);
3099
3100                                         if (opt_showdisassemble) {
3101                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3102                                         }
3103
3104                                         a = 0;
3105
3106                                 } else {
3107                                         a = (ptrint) bte->fp;
3108                                 }
3109
3110                                 d = md->returntype.type;
3111
3112                                 x86_64_mov_imm_reg(cd, a, REG_ITMP1);
3113                                 x86_64_call_reg(cd, REG_ITMP1);
3114                                 break;
3115
3116                         case ICMD_INVOKESPECIAL:
3117                                 x86_64_test_reg_reg(cd, rd->argintregs[0], rd->argintregs[0]);
3118                                 x86_64_jcc(cd, X86_64_CC_Z, 0);
3119                                 codegen_addxnullrefs(cd, cd->mcodeptr);
3120
3121                                 /* first argument contains pointer */
3122 /*                              gen_nullptr_check(rd->argintregs[0]); */
3123
3124                                 /* access memory for hardware nullptr */
3125 /*                              x86_64_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); */
3126
3127                                 /* fall through */
3128
3129                         case ICMD_INVOKESTATIC:
3130                                 if (!lm) {
3131                                         unresolved_method *um = iptr->target;
3132
3133                                         codegen_addpatchref(cd, cd->mcodeptr,
3134                                                                                 PATCHER_invokestatic_special, um, 0);
3135
3136                                         if (opt_showdisassemble) {
3137                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3138                                         }
3139
3140                                         a = 0;
3141                                         d = um->methodref->parseddesc.md->returntype.type;
3142
3143                                 } else {
3144                                         a = (ptrint) lm->stubroutine;
3145                                         d = lm->parseddesc->returntype.type;
3146                                 }
3147
3148                                 x86_64_mov_imm_reg(cd, a, REG_ITMP2);
3149                                 x86_64_call_reg(cd, REG_ITMP2);
3150                                 break;
3151
3152                         case ICMD_INVOKEVIRTUAL:
3153                                 gen_nullptr_check(rd->argintregs[0]);
3154
3155                                 if (!lm) {
3156                                         unresolved_method *um = iptr->target;
3157
3158                                         codegen_addpatchref(cd, cd->mcodeptr,
3159                                                                                 PATCHER_invokevirtual, um, 0);
3160
3161                                         if (opt_showdisassemble) {
3162                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3163                                         }
3164
3165                                         s1 = 0;
3166                                         d = um->methodref->parseddesc.md->returntype.type;
3167
3168                                 } else {
3169                                         s1 = OFFSET(vftbl_t, table[0]) +
3170                                                 sizeof(methodptr) * lm->vftblindex;
3171                                         d = lm->parseddesc->returntype.type;
3172                                 }
3173
3174                                 x86_64_mov_membase_reg(cd, rd->argintregs[0],
3175                                                                            OFFSET(java_objectheader, vftbl),
3176                                                                            REG_ITMP2);
3177                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP1);
3178                                 x86_64_call_reg(cd, REG_ITMP1);
3179                                 break;
3180
3181                         case ICMD_INVOKEINTERFACE:
3182                                 gen_nullptr_check(rd->argintregs[0]);
3183
3184                                 if (!lm) {
3185                                         unresolved_method *um = iptr->target;
3186
3187                                         codegen_addpatchref(cd, cd->mcodeptr,
3188                                                                                 PATCHER_invokeinterface, um, 0);
3189
3190                                         if (opt_showdisassemble) {
3191                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3192                                         }
3193
3194                                         s1 = 0;
3195                                         s2 = 0;
3196                                         d = um->methodref->parseddesc.md->returntype.type;
3197
3198                                 } else {
3199                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3200                                                 sizeof(methodptr) * lm->class->index;
3201
3202                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3203
3204                                         d = lm->parseddesc->returntype.type;
3205                                 }
3206
3207                                 x86_64_mov_membase_reg(cd, rd->argintregs[0],
3208                                                                            OFFSET(java_objectheader, vftbl),
3209                                                                            REG_ITMP2);
3210                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP2);
3211                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s2, REG_ITMP1);
3212                                 x86_64_call_reg(cd, REG_ITMP1);
3213                                 break;
3214                         }
3215
3216                         /* d contains return type */
3217
3218                         if (d != TYPE_VOID) {
3219                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3220                                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3221                                         M_INTMOVE(REG_RESULT, s1);
3222                                         store_reg_to_var_int(iptr->dst, s1);
3223                                 } else {
3224                                         s1 = reg_of_var(rd, iptr->dst, REG_FRESULT);
3225                                         M_FLTMOVE(REG_FRESULT, s1);
3226                                         store_reg_to_var_flt(iptr->dst, s1);
3227                                 }
3228                         }
3229                         break;
3230
3231
3232                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3233
3234                                       /* op1:   0 == array, 1 == class                */
3235                                       /* val.a: (classinfo *) superclass              */
3236
3237                         /*  superclass is an interface:
3238                          *
3239                          *  OK if ((sub == NULL) ||
3240                          *         (sub->vftbl->interfacetablelength > super->index) &&
3241                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3242                          *
3243                          *  superclass is a class:
3244                          *
3245                          *  OK if ((sub == NULL) || (0
3246                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3247                          *         super->vftbl->diffval));
3248                          */
3249
3250                         {
3251                         classinfo *super;
3252                         vftbl_t   *supervftbl;
3253                         s4         superindex;
3254
3255                         super = (classinfo *) iptr->val.a;
3256
3257                         if (!super) {
3258                                 superindex = 0;
3259                                 supervftbl = NULL;
3260
3261                         } else {
3262                                 superindex = super->index;
3263                                 supervftbl = super->vftbl;
3264                         }
3265
3266 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3267             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3268 #endif
3269                         var_to_reg_int(s1, src, REG_ITMP1);
3270
3271                         /* calculate interface checkcast code size */
3272
3273                         s2 = 3; /* mov_membase_reg */
3274                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3275
3276                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3277                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3278                                 3 /* test */ + 6 /* jcc */;
3279
3280                         if (!super)
3281                                 s2 += (opt_showdisassemble ? 5 : 0);
3282
3283                         /* calculate class checkcast code size */
3284
3285                         s3 = 3; /* mov_membase_reg */
3286                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3287                         s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3288
3289 #if 0
3290                         if (s1 != REG_ITMP1) {
3291                                 a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3292                                 CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3293                                 a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3294                                 CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3295                                 a += 3;    /* sub */
3296
3297                         } else
3298 #endif
3299                                 {
3300                                         s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3301                                                 10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3302                                         CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3303                                 }
3304
3305                         s3 += 3 /* cmp */ + 6 /* jcc */;
3306
3307                         if (!super)
3308                                 s3 += (opt_showdisassemble ? 5 : 0);
3309
3310                         /* if class is not resolved, check which code to call */
3311
3312                         if (!super) {
3313                                 x86_64_test_reg_reg(cd, s1, s1);
3314                                 x86_64_jcc(cd, X86_64_CC_Z, 6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3315
3316                                 codegen_addpatchref(cd, cd->mcodeptr,
3317                                                                         PATCHER_checkcast_instanceof_flags,
3318                                                                         (constant_classref *) iptr->target, 0);
3319
3320                                 if (opt_showdisassemble) {
3321                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3322                                 }
3323
3324                                 x86_64_movl_imm_reg(cd, 0, REG_ITMP2);        /* super->flags */
3325                                 x86_64_alul_imm_reg(cd, X86_64_AND, ACC_INTERFACE, REG_ITMP2);
3326                                 x86_64_jcc(cd, X86_64_CC_Z, s2 + 5);
3327                         }
3328
3329                         /* interface checkcast code */
3330
3331                         if (!super || (super->flags & ACC_INTERFACE)) {
3332                                 if (super) {
3333                                         x86_64_test_reg_reg(cd, s1, s1);
3334                                         x86_64_jcc(cd, X86_64_CC_Z, s2);
3335                                 }
3336
3337                                 x86_64_mov_membase_reg(cd, s1,
3338                                                                            OFFSET(java_objectheader, vftbl),
3339                                                                            REG_ITMP2);
3340
3341                                 if (!super) {
3342                                         codegen_addpatchref(cd, cd->mcodeptr,
3343                                                                                 PATCHER_checkcast_instanceof_interface,
3344                                                                                 (constant_classref *) iptr->target, 0);
3345
3346                                         if (opt_showdisassemble) {
3347                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3348                                         }
3349                                 }
3350
3351                                 x86_64_movl_membase32_reg(cd, REG_ITMP2,
3352                                                                                   OFFSET(vftbl_t, interfacetablelength),
3353                                                                                   REG_ITMP3);
3354                                 x86_64_alu_imm32_reg(cd, X86_64_SUB, superindex, REG_ITMP3);
3355                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3356                                 x86_64_jcc(cd, X86_64_CC_LE, 0);
3357                                 codegen_addxcastrefs(cd, cd->mcodeptr);
3358                                 x86_64_mov_membase32_reg(cd, REG_ITMP2,
3359                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3360                                                                                  superindex * sizeof(methodptr*),
3361                                                                                  REG_ITMP3);
3362                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3363                                 x86_64_jcc(cd, X86_64_CC_E, 0);
3364                                 codegen_addxcastrefs(cd, cd->mcodeptr);
3365
3366                                 if (!super)
3367                                         x86_64_jmp_imm(cd, s3);
3368                         }
3369
3370                         /* class checkcast code */
3371
3372                         if (!super || !(super->flags & ACC_INTERFACE)) {
3373                                 if (super) {
3374                                         x86_64_test_reg_reg(cd, s1, s1);
3375                                         x86_64_jcc(cd, X86_64_CC_Z, s3);
3376                                 }
3377
3378                                 x86_64_mov_membase_reg(cd, s1,
3379                                                                            OFFSET(java_objectheader, vftbl),
3380                                                                            REG_ITMP2);
3381
3382                                 if (!super) {
3383                                         codegen_addpatchref(cd, cd->mcodeptr,
3384                                                                                 PATCHER_checkcast_class,
3385                                                                                 (constant_classref *) iptr->target, 0);
3386
3387                                         if (opt_showdisassemble) {
3388                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3389                                         }
3390                                 }
3391
3392                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP3);
3393 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3394                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3395 #endif
3396                                 x86_64_movl_membase32_reg(cd, REG_ITMP2,
3397                                                                                   OFFSET(vftbl_t, baseval),
3398                                                                                   REG_ITMP2);
3399 /*                                      if (s1 != REG_ITMP1) { */
3400 /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3401 /*                                                                                              OFFSET(vftbl_t, baseval), */
3402 /*                                                                                              REG_ITMP1); */
3403 /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3404 /*                                                                                              OFFSET(vftbl_t, diffval), */
3405 /*                                                                                              REG_ITMP3); */
3406 /*  #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
3407 /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3408 /*  #endif */
3409 /*                                              x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP1, REG_ITMP2); */
3410
3411 /*                                      } else { */
3412                                 x86_64_movl_membase32_reg(cd, REG_ITMP3,
3413                                                                                   OFFSET(vftbl_t, baseval),
3414                                                                                   REG_ITMP3);
3415                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP3, REG_ITMP2);
3416                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP3);
3417                                 x86_64_movl_membase_reg(cd, REG_ITMP3,
3418                                                                                 OFFSET(vftbl_t, diffval),
3419                                                                                 REG_ITMP3);
3420 /*                                      } */
3421 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3422                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3423 #endif
3424                                 x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP3, REG_ITMP2);
3425                                 x86_64_jcc(cd, X86_64_CC_A, 0);    /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3426                                 codegen_addxcastrefs(cd, cd->mcodeptr);
3427                         }
3428                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3429                         M_INTMOVE(s1, d);
3430                         store_reg_to_var_int(iptr->dst, d);
3431                         }
3432                         break;
3433
3434                 case ICMD_ARRAYCHECKCAST: /* ..., objectref ==> ..., objectref        */
3435                                           /* op1: 1... resolved, 0... not resolved    */
3436
3437                         var_to_reg_int(s1, src, REG_ITMP1);
3438                         M_INTMOVE(s1, rd->argintregs[0]);
3439
3440                         bte = iptr->val.a;
3441
3442                         if (!iptr->op1) {
3443                                 codegen_addpatchref(cd, cd->mcodeptr, bte->fp, iptr->target, 0);
3444
3445                                 if (opt_showdisassemble) {
3446                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3447                                 }
3448
3449                                 a = 0;
3450
3451                         } else {
3452                                 a = (ptrint) bte->fp;
3453                         }
3454
3455                         x86_64_mov_imm_reg(cd, (ptrint) iptr->target, rd->argintregs[1]);
3456                         x86_64_mov_imm_reg(cd, (ptrint) a, REG_ITMP1);
3457                         x86_64_call_reg(cd, REG_ITMP1);
3458                         M_TEST(REG_RESULT);
3459                         M_BEQ(0);
3460                         codegen_addxcastrefs(cd, cd->mcodeptr);
3461
3462                         var_to_reg_int(s1, src, REG_ITMP1);
3463                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
3464                         M_INTMOVE(s1, d);
3465                         store_reg_to_var_int(iptr->dst, d);
3466                         break;
3467
3468                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3469
3470                                       /* op1:   0 == array, 1 == class                */
3471                                       /* val.a: (classinfo *) superclass              */
3472
3473                         /*  superclass is an interface:
3474                          *
3475                          *  return (sub != NULL) &&
3476                          *         (sub->vftbl->interfacetablelength > super->index) &&
3477                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3478                          *
3479                          *  superclass is a class:
3480                          *
3481                          *  return ((sub != NULL) && (0
3482                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3483                          *          super->vftbl->diffvall));
3484                          */
3485
3486                         {
3487                         classinfo *super;
3488                         vftbl_t   *supervftbl;
3489                         s4         superindex;
3490
3491                         super = (classinfo *) iptr->val.a;
3492
3493                         if (!super) {
3494                                 superindex = 0;
3495                                 supervftbl = NULL;
3496
3497                         } else {
3498                                 superindex = super->index;
3499                                 supervftbl = super->vftbl;
3500                         }
3501
3502 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3503             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3504 #endif
3505
3506                         var_to_reg_int(s1, src, REG_ITMP1);
3507                         d = reg_of_var(rd, iptr->dst, REG_ITMP2);
3508                         if (s1 == d) {
3509                                 M_INTMOVE(s1, REG_ITMP1);
3510                                 s1 = REG_ITMP1;
3511                         }
3512
3513                         /* calculate interface instanceof code size */
3514
3515                         s2 = 3; /* mov_membase_reg */
3516                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3517                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3518                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3519                                 3 /* test */ + 4 /* setcc */;
3520
3521                         if (!super)
3522                                 s2 += (opt_showdisassemble ? 5 : 0);
3523
3524                         /* calculate class instanceof code size */
3525
3526                         s3 = 3; /* mov_membase_reg */
3527                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3528                         s3 += 10; /* mov_imm_reg */
3529                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3530                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3531                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3532                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3533                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3534                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3535                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3536
3537                         if (!super)
3538                                 s3 += (opt_showdisassemble ? 5 : 0);
3539
3540                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3541
3542                         /* if class is not resolved, check which code to call */
3543
3544                         if (!super) {
3545                                 x86_64_test_reg_reg(cd, s1, s1);
3546                                 x86_64_jcc(cd, X86_64_CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3547                                                                                          7 + 6 + s2 + 5 + s3));
3548
3549                                 codegen_addpatchref(cd, cd->mcodeptr,
3550                                                                         PATCHER_checkcast_instanceof_flags,
3551                                                                         (constant_classref *) iptr->target, 0);
3552
3553                                 if (opt_showdisassemble) {
3554                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3555                                 }
3556
3557                                 x86_64_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3558                                 x86_64_alul_imm_reg(cd, X86_64_AND, ACC_INTERFACE, REG_ITMP3);
3559                                 x86_64_jcc(cd, X86_64_CC_Z, s2 + 5);
3560                         }
3561
3562                         /* interface instanceof code */
3563
3564                         if (!super || (super->flags & ACC_INTERFACE)) {
3565                                 if (super) {
3566                                         x86_64_test_reg_reg(cd, s1, s1);
3567                                         x86_64_jcc(cd, X86_64_CC_Z, s2);
3568                                 }
3569
3570                                 x86_64_mov_membase_reg(cd, s1,
3571                                                                            OFFSET(java_objectheader, vftbl),
3572                                                                            REG_ITMP1);
3573                                 if (!super) {
3574                                         codegen_addpatchref(cd, cd->mcodeptr,
3575                                                                                 PATCHER_checkcast_instanceof_interface,
3576                                                                                 (constant_classref *) iptr->target, 0);
3577
3578                                         if (opt_showdisassemble) {
3579                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3580                                         }
3581                                 }
3582
3583                                 x86_64_movl_membase32_reg(cd, REG_ITMP1,
3584                                                                                   OFFSET(vftbl_t, interfacetablelength),
3585                                                                                   REG_ITMP3);
3586                                 x86_64_alu_imm32_reg(cd, X86_64_SUB, superindex, REG_ITMP3);
3587                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3588
3589                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3590
3591                                 x86_64_jcc(cd, X86_64_CC_LE, a);
3592                                 x86_64_mov_membase32_reg(cd, REG_ITMP1,
3593                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3594                                                                                  superindex * sizeof(methodptr*),
3595                                                                                  REG_ITMP1);
3596                                 x86_64_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3597                                 x86_64_setcc_reg(cd, X86_64_CC_NE, d);
3598
3599                                 if (!super)
3600                                         x86_64_jmp_imm(cd, s3);
3601                         }
3602
3603                         /* class instanceof code */
3604
3605                         if (!super || !(super->flags & ACC_INTERFACE)) {
3606                                 if (super) {
3607                                         x86_64_test_reg_reg(cd, s1, s1);
3608                                         x86_64_jcc(cd, X86_64_CC_E, s3);
3609                                 }
3610
3611                                 x86_64_mov_membase_reg(cd, s1,
3612                                                                            OFFSET(java_objectheader, vftbl),
3613                                                                            REG_ITMP1);
3614
3615                                 if (!super) {
3616                                         codegen_addpatchref(cd, cd->mcodeptr,
3617                                                                                 PATCHER_instanceof_class,
3618                                                                                 (constant_classref *) iptr->target, 0);
3619
3620                                         if (opt_showdisassemble) {
3621                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3622                                         }
3623                                 }
3624
3625                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3626 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3627                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3628 #endif
3629                                 x86_64_movl_membase_reg(cd, REG_ITMP1,
3630                                                                                 OFFSET(vftbl_t, baseval),
3631                                                                                 REG_ITMP1);
3632                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3633                                                                                 OFFSET(vftbl_t, diffval),
3634                                                                                 REG_ITMP3);
3635                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3636                                                                                 OFFSET(vftbl_t, baseval),
3637                                                                                 REG_ITMP2);
3638 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3639                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3640 #endif
3641                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
3642                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d); /* may be REG_ITMP2 */
3643                                 x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP3, REG_ITMP1);
3644                                 x86_64_setcc_reg(cd, X86_64_CC_BE, d);
3645                         }
3646                         store_reg_to_var_int(iptr->dst, d);
3647                         }
3648                         break;
3649
3650                 case ICMD_CHECKASIZE:  /* ..., size ==> ..., size                     */
3651
3652                         if (src->flags & INMEMORY) {
3653                                 x86_64_alul_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
3654
3655                         } else {
3656                                 x86_64_testl_reg_reg(cd, src->regoff, src->regoff);
3657                         }
3658                         x86_64_jcc(cd, X86_64_CC_L, 0);
3659                         codegen_addxcheckarefs(cd, cd->mcodeptr);
3660                         break;
3661
3662                 case ICMD_CHECKEXCEPTION:    /* ... ==> ...                           */
3663
3664                         x86_64_test_reg_reg(cd, REG_RESULT, REG_RESULT);
3665                         x86_64_jcc(cd, X86_64_CC_E, 0);
3666                         codegen_addxexceptionrefs(cd, cd->mcodeptr);
3667                         break;
3668
3669                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3670                                          /* op1 = dimension, val.a = array descriptor */
3671
3672                         /* check for negative sizes and copy sizes to stack if necessary  */
3673
3674                         MCODECHECK((10 * 4 * iptr->op1) + 5 + 10 * 8);
3675
3676                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3677                                 var_to_reg_int(s2, src, REG_ITMP1);
3678                                 x86_64_testl_reg_reg(cd, s2, s2);
3679                                 x86_64_jcc(cd, X86_64_CC_L, 0);
3680                                 codegen_addxcheckarefs(cd, cd->mcodeptr);
3681
3682                                 /* copy SAVEDVAR sizes to stack */
3683
3684                                 if (src->varkind != ARGVAR) {
3685                                         x86_64_mov_reg_membase(cd, s2, REG_SP, s1 * 8);
3686                                 }
3687                         }
3688
3689                         /* is a patcher function set? */
3690
3691                         if (iptr->target) {
3692                                 codegen_addpatchref(cd, cd->mcodeptr,
3693                                                                         (functionptr) (ptrint) iptr->target,
3694                                                                         iptr->val.a, 0);
3695
3696                                 if (opt_showdisassemble) {
3697                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3698                                 }
3699
3700                                 a = 0;
3701
3702                         } else {
3703                                 a = (ptrint) iptr->val.a;
3704                         }
3705
3706                         /* a0 = dimension count */
3707
3708                         x86_64_mov_imm_reg(cd, iptr->op1, rd->argintregs[0]);
3709
3710                         /* a1 = arrayvftbl */
3711
3712                         x86_64_mov_imm_reg(cd, (ptrint) iptr->val.a, rd->argintregs[1]);
3713
3714                         /* a2 = pointer to dimensions = stack pointer */
3715
3716                         x86_64_mov_reg_reg(cd, REG_SP, rd->argintregs[2]);
3717
3718                         x86_64_mov_imm_reg(cd, (ptrint) BUILTIN_multianewarray, REG_ITMP1);
3719                         x86_64_call_reg(cd, REG_ITMP1);
3720
3721                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3722                         M_INTMOVE(REG_RESULT, s1);
3723                         store_reg_to_var_int(iptr->dst, s1);
3724                         break;
3725
3726                 default:
3727                         throw_cacao_exception_exit(string_java_lang_InternalError,
3728                                                                            "Unknown ICMD %d", iptr->opc);
3729         } /* switch */
3730
3731         } /* for instruction */
3732
3733         /* copy values to interface registers */
3734
3735         src = bptr->outstack;
3736         len = bptr->outdepth;
3737         MCODECHECK(512);
3738 #ifdef LSRA
3739         if (!opt_lsra)
3740 #endif
3741         while (src) {
3742                 len--;
3743                 if ((src->varkind != STACKVAR)) {
3744                         s2 = src->type;
3745                         if (IS_FLT_DBL_TYPE(s2)) {
3746                                 var_to_reg_flt(s1, src, REG_FTMP1);
3747                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3748                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3749
3750                                 } else {
3751                                         x86_64_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3752                                 }
3753
3754                         } else {
3755                                 var_to_reg_int(s1, src, REG_ITMP1);
3756                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3757                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3758
3759                                 } else {
3760                                         x86_64_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3761                                 }
3762                         }
3763                 }
3764                 src = src->prev;
3765         }
3766         } /* if (bptr -> flags >= BBREACHED) */
3767         } /* for basic block */
3768
3769         codegen_createlinenumbertable(cd);
3770
3771         {
3772
3773         u1        *xcodeptr;
3774         branchref *bref;
3775
3776         /* generate ArithmeticException stubs */
3777
3778         xcodeptr = NULL;
3779
3780         for (bref = cd->xdivrefs; bref != NULL; bref = bref->next) {
3781                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3782                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3783                                                           bref->branchpos,
3784                                                           xcodeptr - cd->mcodebase - (10 + 7));
3785                         continue;
3786                 }
3787
3788                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3789                                   bref->branchpos,
3790                                                   cd->mcodeptr - cd->mcodebase);
3791
3792                 MCODECHECK(512);
3793
3794                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3795                 dseg_adddata(cd, cd->mcodeptr);
3796                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3797
3798                 if (xcodeptr != NULL) {
3799                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3800
3801                 } else {
3802                         xcodeptr = cd->mcodeptr;
3803
3804                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3805                         M_MOV(REG_SP, rd->argintregs[1]);
3806                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3807                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3808
3809                         M_ASUB_IMM(2 * 8, REG_SP);
3810                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3811
3812                         M_MOV_IMM((ptrint) stacktrace_inline_arithmeticexception,
3813                                           REG_ITMP3);
3814                         M_CALL(REG_ITMP3);
3815
3816                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3817                         M_AADD_IMM(2 * 8, REG_SP);
3818
3819                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3820                         M_JMP(REG_ITMP3);
3821                 }
3822         }
3823
3824         /* generate ArrayIndexOutOfBoundsException stubs */
3825
3826         xcodeptr = NULL;
3827
3828         for (bref = cd->xboundrefs; bref != NULL; bref = bref->next) {
3829                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3830                                   bref->branchpos,
3831                                                   cd->mcodeptr - cd->mcodebase);
3832
3833                 MCODECHECK(512);
3834
3835                 /* move index register into REG_ITMP1 */
3836
3837                 M_MOV(bref->reg, REG_ITMP1);                              /* 3 bytes  */
3838
3839                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3840                 dseg_adddata(cd, cd->mcodeptr);
3841                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3842
3843                 if (xcodeptr != NULL) {
3844                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3845
3846                 } else {
3847                         xcodeptr = cd->mcodeptr;
3848
3849                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3850                         M_MOV(REG_SP, rd->argintregs[1]);
3851                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3852                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3853                         M_MOV(REG_ITMP1, rd->argintregs[4]);
3854
3855                         M_ASUB_IMM(2 * 8, REG_SP);
3856                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3857
3858                         M_MOV_IMM((ptrint) stacktrace_inline_arrayindexoutofboundsexception,
3859                                           REG_ITMP3);
3860                         M_CALL(REG_ITMP3);
3861
3862                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3863                         M_AADD_IMM(2 * 8, REG_SP);
3864
3865                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3866                         M_JMP(REG_ITMP3);
3867                 }
3868         }
3869
3870         /* generate ArrayStoreException stubs */
3871
3872         xcodeptr = NULL;
3873
3874         for (bref = cd->xstorerefs; bref != NULL; bref = bref->next) {
3875                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3876                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3877                                                           bref->branchpos,
3878                                                           xcodeptr - cd->mcodebase - (10 + 7));
3879                         continue;
3880                 }
3881
3882                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3883                                   bref->branchpos,
3884                                                   cd->mcodeptr - cd->mcodebase);
3885
3886                 MCODECHECK(512);
3887
3888                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3889                 dseg_adddata(cd, cd->mcodeptr);
3890                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3891
3892                 if (xcodeptr != NULL) {
3893                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3894
3895                 } else {
3896                         xcodeptr = cd->mcodeptr;
3897
3898                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3899                         M_MOV(REG_SP, rd->argintregs[1]);
3900                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3901                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3902
3903                         M_ASUB_IMM(2 * 8, REG_SP);
3904                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3905
3906                         M_MOV_IMM((ptrint) stacktrace_inline_arraystoreexception,
3907                                           REG_ITMP3);
3908                         M_CALL(REG_ITMP3);
3909
3910                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3911                         M_AADD_IMM(2 * 8, REG_SP);
3912
3913                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3914                         M_JMP(REG_ITMP3);
3915                 }
3916         }
3917
3918         /* generate ClassCastException stubs */
3919
3920         xcodeptr = NULL;
3921
3922         for (bref = cd->xcastrefs; bref != NULL; bref = bref->next) {
3923                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3924                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3925                                                           bref->branchpos,
3926                                                           xcodeptr - cd->mcodebase - (10 + 7));
3927                         continue;
3928                 }
3929
3930                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3931                                   bref->branchpos,
3932                                                   cd->mcodeptr - cd->mcodebase);
3933
3934                 MCODECHECK(512);
3935
3936                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3937                 dseg_adddata(cd, cd->mcodeptr);
3938                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3939
3940                 if (xcodeptr != NULL) {
3941                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3942
3943                 } else {
3944                         xcodeptr = cd->mcodeptr;
3945
3946                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3947                         M_MOV(REG_SP, rd->argintregs[1]);
3948                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3949                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3950
3951                         M_ASUB_IMM(2 * 8, REG_SP);
3952                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3953
3954                         M_MOV_IMM((ptrint) stacktrace_inline_classcastexception, REG_ITMP3);
3955                         M_CALL(REG_ITMP3);
3956
3957                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3958                         M_AADD_IMM(2 * 8, REG_SP);
3959
3960                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3961                         M_JMP(REG_ITMP3);
3962                 }
3963         }
3964
3965         /* generate NegativeArraySizeException stubs */
3966
3967         xcodeptr = NULL;
3968
3969         for (bref = cd->xcheckarefs; bref != NULL; bref = bref->next) {
3970                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3971                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3972                                                           bref->branchpos,
3973                                                           xcodeptr - cd->mcodebase - (10 + 7));
3974                         continue;
3975                 }
3976
3977                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3978                                   bref->branchpos,
3979                                                   cd->mcodeptr - cd->mcodebase);
3980
3981                 MCODECHECK(512);
3982
3983                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3984                 dseg_adddata(cd, cd->mcodeptr);
3985                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3986
3987                 if (xcodeptr != NULL) {
3988                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3989
3990                 } else {
3991                         xcodeptr = cd->mcodeptr;
3992
3993                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3994                         M_MOV(REG_SP, rd->argintregs[1]);
3995                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3996                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3997
3998                         M_ASUB_IMM(2 * 8, REG_SP);
3999                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
4000
4001                         M_MOV_IMM((ptrint) stacktrace_inline_negativearraysizeexception,
4002                                           REG_ITMP3);
4003                         M_CALL(REG_ITMP3);
4004
4005                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
4006                         M_AADD_IMM(2 * 8, REG_SP);
4007
4008                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
4009                         M_JMP(REG_ITMP3);
4010                 }
4011         }
4012
4013         /* generate NullpointerException stubs */
4014
4015         xcodeptr = NULL;
4016
4017         for (bref = cd->xnullrefs; bref != NULL; bref = bref->next) {
4018                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
4019                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
4020                                                           bref->branchpos,
4021                                                           xcodeptr - cd->mcodebase - (10 + 7));
4022                         continue;
4023                 }
4024
4025                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
4026                                   bref->branchpos,
4027                                                   cd->mcodeptr - cd->mcodebase);
4028
4029                 MCODECHECK(512);
4030
4031                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
4032                 dseg_adddata(cd, cd->mcodeptr);
4033                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
4034
4035                 if (xcodeptr != NULL) {
4036                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
4037
4038                 } else {
4039                         xcodeptr = cd->mcodeptr;
4040
4041                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
4042                         M_MOV(REG_SP, rd->argintregs[1]);
4043                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
4044                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
4045
4046                         M_ASUB_IMM(2 * 8, REG_SP);
4047                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
4048
4049                         M_MOV_IMM((ptrint) stacktrace_inline_nullpointerexception,
4050                                           REG_ITMP3);
4051                         M_CALL(REG_ITMP3);
4052
4053                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
4054                         M_AADD_IMM(2 * 8, REG_SP);
4055
4056                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
4057                         M_JMP(REG_ITMP3);
4058                 }
4059         }
4060
4061         /* generate ICMD_CHECKEXCEPTION stubs */
4062
4063         xcodeptr = NULL;
4064
4065         for (bref = cd->xexceptionrefs; bref != NULL; bref = bref->next) {
4066                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
4067                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
4068                                                           bref->branchpos,
4069                                                           xcodeptr - cd->mcodebase - (10 + 7));
4070                         continue;
4071                 }
4072
4073                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
4074                                   bref->branchpos,
4075                                                   cd->mcodeptr - cd->mcodebase);
4076
4077                 MCODECHECK(512);
4078
4079                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
4080                 dseg_adddata(cd, cd->mcodeptr);
4081                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
4082
4083                 if (xcodeptr != NULL) {
4084                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
4085
4086                 } else {
4087                         xcodeptr = cd->mcodeptr;
4088
4089                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
4090                         M_MOV(REG_SP, rd->argintregs[1]);
4091                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
4092                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
4093
4094                         M_ASUB_IMM(2 * 8, REG_SP);
4095                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
4096
4097                         M_MOV_IMM((ptrint) stacktrace_inline_fillInStackTrace, REG_ITMP3);
4098                         M_CALL(REG_ITMP3);
4099
4100                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
4101                         M_AADD_IMM(2 * 8, REG_SP);
4102
4103                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
4104                         M_JMP(REG_ITMP3);
4105                 }
4106         }
4107
4108         /* generate code patching stub call code */
4109
4110         {
4111                 patchref    *pref;
4112                 codegendata *tmpcd;
4113                 ptrint       mcode;
4114
4115                 tmpcd = DNEW(codegendata);
4116
4117                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4118                         /* check size of code segment */
4119
4120                         MCODECHECK(512);
4121
4122                         /* Get machine code which is patched back in later. A             */
4123                         /* `call rel32' is 5 bytes long (but read 8 bytes).               */
4124
4125                         xcodeptr = cd->mcodebase + pref->branchpos;
4126                         mcode = *((ptrint *) xcodeptr);
4127
4128                         /* patch in `call rel32' to call the following code               */
4129
4130                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
4131                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
4132
4133                         /* move pointer to java_objectheader onto stack */
4134
4135 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4136                         /* create a virtual java_objectheader */
4137
4138                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4139                         a = dseg_addaddress(cd, NULL);                      /* vftbl      */
4140
4141                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP3);
4142                         M_PUSH(REG_ITMP3);
4143 #else
4144                         M_PUSH_IMM(0);
4145 #endif
4146
4147                         /* move machine code bytes and classinfo pointer into registers */
4148
4149                         M_MOV_IMM((ptrint) mcode, REG_ITMP3);
4150                         M_PUSH(REG_ITMP3);
4151                         M_MOV_IMM((ptrint) pref->ref, REG_ITMP3);
4152                         M_PUSH(REG_ITMP3);
4153                         M_MOV_IMM((ptrint) pref->disp, REG_ITMP3);
4154                         M_PUSH(REG_ITMP3);
4155
4156                         M_MOV_IMM((ptrint) pref->patcher, REG_ITMP3);
4157                         M_PUSH(REG_ITMP3);
4158
4159                         M_MOV_IMM((ptrint) asm_wrapper_patcher, REG_ITMP3);
4160                         M_JMP(REG_ITMP3);
4161                 }
4162         }
4163         }
4164
4165         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
4166 }
4167
4168
4169 /* createcompilerstub **********************************************************
4170
4171    Creates a stub routine which calls the compiler.
4172
4173 *******************************************************************************/
4174
4175 #define COMPILERSTUB_SIZE    23
4176
4177 functionptr createcompilerstub(methodinfo *m)
4178 {
4179         u1          *s;                     /* memory to hold the stub            */
4180         codegendata *cd;
4181         s4           dumpsize;
4182
4183         s = CNEW(u1, COMPILERSTUB_SIZE);
4184
4185         /* mark start of dump memory area */
4186
4187         dumpsize = dump_size();
4188
4189         cd = DNEW(codegendata);
4190         cd->mcodeptr = s;
4191
4192         /* code for the stub */
4193
4194         x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP1); /* pass method to compiler */
4195         x86_64_mov_imm_reg(cd, (ptrint) asm_call_jit_compiler, REG_ITMP3);
4196         x86_64_jmp_reg(cd, REG_ITMP3);
4197
4198 #if defined(STATISTICS)
4199         if (opt_stat)
4200                 count_cstub_len += COMPILERSTUB_SIZE;
4201 #endif
4202
4203         /* release dump area */
4204
4205         dump_release(dumpsize);
4206
4207         return (functionptr) (ptrint) s;
4208 }
4209
4210
4211 /* createnativestub ************************************************************
4212
4213    Creates a stub routine which calls a native method.
4214
4215 *******************************************************************************/
4216
4217 functionptr createnativestub(functionptr f, methodinfo *m, codegendata *cd,
4218                                                          registerdata *rd, methoddesc *nmd)
4219 {
4220         methoddesc *md;
4221         s4          stackframesize;         /* size of stackframe if needed       */
4222         s4          nativeparams;
4223         s4          i, j;                   /* count variables                    */
4224         s4          t;
4225         s4          s1, s2, disp;
4226
4227         /* initialize variables */
4228
4229         md = m->parseddesc;
4230         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
4231
4232
4233         /* calculate stack frame size */
4234
4235         stackframesize =
4236                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
4237                 INT_ARG_CNT + FLT_ARG_CNT + 1 +         /* + 1 for function address   */
4238                 nmd->memuse;
4239
4240         if (!(stackframesize & 0x1))                /* keep stack 16-byte aligned */
4241                 stackframesize++;
4242
4243
4244         /* create method header */
4245
4246         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
4247         (void) dseg_adds4(cd, stackframesize * 8);              /* FrameSize      */
4248         (void) dseg_adds4(cd, 0);                               /* IsSync         */
4249         (void) dseg_adds4(cd, 0);                               /* IsLeaf         */
4250         (void) dseg_adds4(cd, 0);                               /* IntSave        */
4251         (void) dseg_adds4(cd, 0);                               /* FltSave        */
4252         (void) dseg_addlinenumbertablesize(cd);
4253         (void) dseg_adds4(cd, 0);                               /* ExTableSize    */
4254
4255
4256         /* initialize mcode variables */
4257
4258         cd->mcodeptr = (u1 *) cd->mcodebase;
4259         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
4260
4261
4262         /* generate stub code */
4263
4264         M_ASUB_IMM(stackframesize * 8, REG_SP);
4265
4266         if (runverbose) {
4267                 /* save integer and float argument registers */
4268
4269                 for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4270                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4271                                 M_LST(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4272
4273                 for (i = 0, j = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4274                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4275                                 M_DST(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4276
4277                 /* show integer hex code for float arguments */
4278
4279                 for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++) {
4280                         /* if the paramtype is a float, we have to right shift all        */
4281                         /* following integer registers                                    */
4282
4283                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type)) {
4284                                 for (s1 = INT_ARG_CNT - 2; s1 >= i; s1--)
4285                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
4286
4287                                 x86_64_movd_freg_reg(cd, rd->argfltregs[j], rd->argintregs[i]);
4288                                 j++;
4289                         }
4290                 }
4291
4292                 x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP1);
4293                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, 0 * 8);
4294                 x86_64_mov_imm_reg(cd, (ptrint) builtin_trace_args, REG_ITMP1);
4295                 x86_64_call_reg(cd, REG_ITMP1);
4296
4297                 /* restore integer and float argument registers */
4298
4299                 for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4300                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4301                                 M_LLD(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4302
4303                 for (i = 0, j = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4304                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4305                                 M_DLD(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4306         }
4307
4308
4309         /* get function address (this must happen before the stackframeinfo) */
4310
4311 #if !defined(ENABLE_STATICVM)
4312         if (f == NULL) {
4313                 codegen_addpatchref(cd, cd->mcodeptr, PATCHER_resolve_native, m, 0);
4314
4315                 if (opt_showdisassemble) {
4316                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
4317                 }
4318         }
4319 #endif
4320
4321         M_MOV_IMM((ptrint) f, REG_ITMP3);
4322
4323
4324         /* save integer and float argument registers */
4325
4326         for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4327                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4328                         M_LST(rd->argintregs[j++], REG_SP, i * 8);
4329
4330         for (i = 0, j = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4331                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4332                         M_DST(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4333
4334         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4335
4336         /* create dynamic stack info */
4337
4338         M_ALEA(REG_SP, stackframesize * 8 - sizeof(stackframeinfo), rd->argintregs[0]);
4339         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[1]);
4340         M_ALEA(REG_SP, stackframesize * 8 + SIZEOF_VOID_P, rd->argintregs[2]);
4341         M_ALD(rd->argintregs[3], REG_SP, stackframesize * 8);
4342         x86_64_mov_imm_reg(cd, (ptrint) stacktrace_create_native_stackframeinfo,
4343                                            REG_ITMP1);
4344         x86_64_call_reg(cd, REG_ITMP1);
4345
4346         STATS({
4347                 x86_64_mov_imm_reg(cd, (ptrint) nativeinvokation, REG_ITMP1);
4348                 x86_64_call_reg(cd, REG_ITMP1);
4349         })
4350
4351         /* restore integer and float argument registers */
4352
4353         for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4354                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4355                         M_LLD(rd->argintregs[j++], REG_SP, i * 8);
4356
4357         for (i = 0, j = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4358                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4359                         M_DLD(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4360
4361         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4362
4363
4364         /* copy or spill arguments to new locations */
4365
4366         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
4367                 t = md->paramtypes[i].type;
4368
4369                 if (IS_INT_LNG_TYPE(t)) {
4370                         if (!md->params[i].inmemory) {
4371                                 s1 = rd->argintregs[md->params[i].regoff];
4372
4373                                 if (!nmd->params[j].inmemory) {
4374                                         s2 = rd->argintregs[nmd->params[j].regoff];
4375                                         M_INTMOVE(s1, s2);
4376
4377                                 } else {
4378                                         s2 = nmd->params[j].regoff;
4379                                         M_LST(s1, REG_SP, s2 * 8);
4380                                 }
4381
4382                         } else {
4383                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4384                                 s2 = nmd->params[j].regoff;
4385                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
4386                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
4387                         }
4388
4389                 } else {
4390                         /* We only copy spilled float arguments, as the float argument    */
4391                         /* registers keep unchanged.                                      */
4392
4393                         if (md->params[i].inmemory) {
4394                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4395                                 s2 = nmd->params[j].regoff;
4396                                 M_DLD(REG_FTMP1, REG_SP, s1 * 8);
4397                                 M_DST(REG_FTMP1, REG_SP, s2 * 8);
4398                         }
4399                 }
4400         }
4401
4402         /* put class into second argument register */
4403
4404         if (m->flags & ACC_STATIC)
4405                 M_MOV_IMM((ptrint) m->class, rd->argintregs[1]);
4406
4407         /* put env into first argument register */
4408
4409         M_MOV_IMM((ptrint) &env, rd->argintregs[0]);
4410
4411         /* do the native function call */
4412
4413         M_CALL(REG_ITMP3);
4414
4415
4416         /* remove native stackframe info */
4417
4418         if (IS_INT_LNG_TYPE(md->returntype.type))
4419                 M_LST(REG_RESULT, REG_SP, 0 * 8);
4420         else
4421                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
4422
4423         M_ALEA(REG_SP, stackframesize * 8 - sizeof(stackframeinfo), rd->argintregs[0]);
4424         x86_64_mov_imm_reg(cd, (ptrint) stacktrace_remove_stackframeinfo,
4425                                            REG_ITMP1);
4426         x86_64_call_reg(cd, REG_ITMP1);
4427
4428         if (IS_INT_LNG_TYPE(md->returntype.type))
4429                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
4430         else
4431                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4432
4433
4434         /* generate call trace */
4435
4436         if (runverbose) {
4437                 M_LST(REG_RESULT, REG_SP, 0 * 8);
4438                 M_DST(REG_FRESULT, REG_SP, 1 * 8);
4439
4440                 x86_64_mov_imm_reg(cd, (ptrint) m, rd->argintregs[0]);
4441                 M_MOV(REG_RESULT, rd->argintregs[1]);
4442                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4443                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4444
4445                 x86_64_mov_imm_reg(cd, (ptrint) builtin_displaymethodstop, REG_ITMP1);
4446                 x86_64_call_reg(cd, REG_ITMP1);
4447
4448                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
4449                 M_DLD(REG_FRESULT, REG_SP, 1 * 8);
4450         }
4451
4452         /* check for exception */
4453
4454 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4455         M_LST(REG_RESULT, REG_SP, 0 * 8);
4456         x86_64_mov_imm_reg(cd, (ptrint) builtin_get_exceptionptrptr, REG_ITMP3);
4457         x86_64_call_reg(cd, REG_ITMP3);
4458         M_LLD(REG_ITMP3, REG_RESULT, 0);
4459         M_LLD(REG_RESULT, REG_SP, 0 * 8);
4460 #else
4461         x86_64_mov_imm_reg(cd, (ptrint) &_exceptionptr, REG_ITMP3);
4462         x86_64_mov_membase_reg(cd, REG_ITMP3, 0, REG_ITMP3);
4463 #endif
4464         x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
4465         x86_64_jcc(cd, X86_64_CC_NE, 7 + 1);
4466
4467         /* remove stackframe */
4468
4469         M_AADD_IMM(stackframesize * 8, REG_SP);
4470
4471         x86_64_ret(cd);
4472
4473
4474         /* handle exception */
4475
4476 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4477         M_LST(REG_ITMP3, REG_SP, 0 * 8);
4478         x86_64_mov_imm_reg(cd, (ptrint) builtin_get_exceptionptrptr, REG_ITMP3);
4479         x86_64_call_reg(cd, REG_ITMP3);
4480         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
4481         M_LLD(REG_ITMP1_XPTR, REG_SP, 0 * 8);
4482 #else
4483         x86_64_mov_reg_reg(cd, REG_ITMP3, REG_ITMP1_XPTR);
4484         x86_64_mov_imm_reg(cd, (ptrint) &_exceptionptr, REG_ITMP3);
4485         x86_64_alu_reg_reg(cd, X86_64_XOR, REG_ITMP2, REG_ITMP2);
4486         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_ITMP3, 0);    /* clear exception pointer */
4487 #endif
4488
4489         /* remove stackframe */
4490
4491         M_AADD_IMM(stackframesize * 8, REG_SP);
4492
4493         M_LLD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
4494         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
4495
4496         x86_64_mov_imm_reg(cd, (ptrint) asm_handle_nat_exception, REG_ITMP3);
4497         x86_64_jmp_reg(cd, REG_ITMP3);
4498
4499
4500         /* process patcher calls **************************************************/
4501
4502         {
4503                 u1          *xcodeptr;
4504                 patchref    *pref;
4505                 codegendata *tmpcd;
4506                 ptrint       mcode;
4507
4508                 tmpcd = DNEW(codegendata);
4509
4510                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4511                         /* Get machine code which is patched back in later. A             */
4512                         /* `call rel32' is 5 bytes long (but read 8 bytes).               */
4513
4514                         xcodeptr = cd->mcodebase + pref->branchpos;
4515                         mcode = *((ptrint *) xcodeptr);
4516
4517                         /* patch in `call rel32' to call the following code               */
4518
4519                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
4520                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
4521
4522                         /* move pointer to java_objectheader onto stack */
4523
4524 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4525                         /* create a virtual java_objectheader */
4526
4527                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4528                         disp = dseg_addaddress(cd, NULL);                   /* vftbl      */
4529
4530                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP3);
4531                         M_PUSH(REG_ITMP3);
4532 #else
4533                         M_PUSH_IMM(0);
4534 #endif
4535
4536                         /* move machine code bytes and classinfo pointer into registers */
4537
4538                         M_MOV_IMM((ptrint) mcode, REG_ITMP3);
4539                         M_PUSH(REG_ITMP3);
4540                         M_MOV_IMM((ptrint) pref->ref, REG_ITMP3);
4541                         M_PUSH(REG_ITMP3);
4542                         M_MOV_IMM((ptrint) pref->disp, REG_ITMP3);
4543                         M_PUSH(REG_ITMP3);
4544
4545                         M_MOV_IMM((ptrint) pref->patcher, REG_ITMP3);
4546                         M_PUSH(REG_ITMP3);
4547
4548                         M_MOV_IMM((ptrint) asm_wrapper_patcher, REG_ITMP3);
4549                         M_JMP(REG_ITMP3);
4550                 }
4551         }
4552
4553         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
4554
4555         return m->entrypoint;
4556 }
4557
4558
4559 /*
4560  * These are local overrides for various environment variables in Emacs.
4561  * Please do not remove this and leave it at the end of the file, where
4562  * Emacs will automagically detect them.
4563  * ---------------------------------------------------------------------
4564  * Local variables:
4565  * mode: c
4566  * indent-tabs-mode: t
4567  * c-basic-offset: 4
4568  * tab-width: 4
4569  * End:
4570  */