src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005 R. Grafl, A. Krall, C. Kruegel, C. Oates,
   4    R. Obermaisser, M. Platter, M. Probst, S. Ring, E. Steiner,
   5    C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich, J. Wenninger,
   6    Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  23    02111-1307, USA.
  24
  25    Contact: cacao@complang.tuwien.ac.at
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31
  32    $Id: codegen.c 2848 2005-06-27 20:59:28Z twisti $
  33
  34 */
  35
  36
  37 #include <stdio.h>
  38
  39 #include "config.h"
  40
  41 #include "md.h"
  42 #include "md-abi.h"
  43 #include "md-abi.inc"
  44
  45 #include "vm/jit/x86_64/arch.h"
  46 #include "vm/jit/x86_64/codegen.h"
  47 #include "vm/jit/x86_64/emitfuncs.h"
  48 #include "vm/jit/x86_64/types.h"
  49 #include "vm/jit/x86_64/asmoffsets.h"
  50
  51 #include "cacao/cacao.h"
  52 #include "native/native.h"
  53 #include "vm/global.h"
  54 #include "vm/builtin.h"
  55 #include "vm/loader.h"
  56 #include "vm/statistics.h"
  57 #include "vm/stringlocal.h"
  58 #include "vm/tables.h"
  59 #include "vm/jit/asmpart.h"
  60 #include "vm/jit/codegen.inc"
  61 #include "vm/jit/helper.h"
  62 #include "vm/jit/jit.h"
  63
  64 #if defined(LSRA)
  65 # include "vm/jit/lsra.inc"
  66 #endif
  67
  68 #include "vm/jit/parse.h"
  69 #include "vm/jit/patcher.h"
  70 #include "vm/jit/reg.h"
  71 #include "vm/jit/reg.inc"
  72
  73
  74 /* codegen *********************************************************************
  75
  76    Generates machine code.
  77
  78 *******************************************************************************/
  79
  80 void codegen(methodinfo *m, codegendata *cd, registerdata *rd)
  81 {
  82         s4                  len, s1, s2, s3, d;
  83         u2                  currentline;
  84         ptrint              a;
  85         s4                  parentargs_base;
  86         stackptr            src;
  87         varinfo            *var;
  88         basicblock         *bptr;
  89         instruction        *iptr;
  90         exceptiontable     *ex;
  91         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
  92         builtintable_entry *bte;
  93         methoddesc         *md;
  94
  95         {
  96         s4 i, p, t, l;
  97         s4 savedregs_num;
  98
  99         savedregs_num = 0;
 100
 101         /* space to save used callee saved registers */
 102
 103         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 104         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 105
 106         parentargs_base = rd->memuse + savedregs_num;
 107
 108 #if defined(USE_THREADS)           /* space to save argument of monitor_enter */
 109
 110         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 111                 parentargs_base++;
 112
 113 #endif
 114
 115     /* Keep stack of non-leaf functions 16-byte aligned for calls into native */
 116         /* code e.g. libc or jni (alignment problems with movaps).                */
 117
 118         if (!m->isleafmethod || runverbose)
 119                 parentargs_base |= 0x1;
 120
 121         /* create method header */
 122
 123         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 124         (void) dseg_adds4(cd, parentargs_base * 8);             /* FrameSize      */
 125
 126 #if defined(USE_THREADS)
 127
 128         /* IsSync contains the offset relative to the stack pointer for the
 129            argument of monitor_exit used in the exception handler. Since the
 130            offset could be zero and give a wrong meaning of the flag it is
 131            offset by one.
 132         */
 133
 134         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 135                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);     /* IsSync         */
 136         else
 137
 138 #endif
 139
 140                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 141
 142         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 143         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse);/* IntSave  */
 144         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse);/* FltSave  */
 145
 146         (void) dseg_addlinenumbertablesize(cd);
 147
 148         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 149
 150         /* create exception table */
 151
 152         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 153                 dseg_addtarget(cd, ex->start);
 154                 dseg_addtarget(cd, ex->end);
 155                 dseg_addtarget(cd, ex->handler);
 156                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 157         }
 158
 159         /* initialize mcode variables */
 160
 161         cd->mcodeptr = (u1 *) cd->mcodebase;
 162         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
 163         MCODECHECK(512);
 164
 165         /* create stack frame (if necessary) */
 166
 167         if (parentargs_base) {
 168                 x86_64_alu_imm_reg(cd, X86_64_SUB, parentargs_base * 8, REG_SP);
 169         }
 170
 171         /* save used callee saved registers */
 172
 173         p = parentargs_base;
 174         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 175                 p--; x86_64_mov_reg_membase(cd, rd->savintregs[i], REG_SP, p * 8);
 176         }
 177         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 178                 p--; x86_64_movq_reg_membase(cd, rd->savfltregs[i], REG_SP, p * 8);
 179         }
 180
 181         /* take arguments out of register or stack frame */
 182
 183         md = m->parseddesc;
 184
 185         for (p = 0, l = 0; p < md->paramcount; p++) {
 186                 t = md->paramtypes[p].type;
 187                 var = &(rd->locals[l][t]);
 188                 l++;
 189                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 190                         l++;
 191                 if (var->type < 0)
 192                         continue;
 193                 s1 = md->params[p].regoff;
 194                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 195                         s2 = rd->argintregs[s1];
 196                         if (!md->params[p].inmemory) {           /* register arguments    */
 197                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 198                                         M_INTMOVE(s2, var->regoff);
 199
 200                                 } else {                             /* reg arg -> spilled    */
 201                                     M_LST(s2, REG_SP, var->regoff * 8);
 202                                 }
 203
 204                         } else {                                 /* stack arguments       */
 205                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 206                                         /* + 8 for return address */
 207                                         M_LLD(var->regoff, REG_SP, (parentargs_base + s1) * 8 + 8);
 208
 209                                 } else {                             /* stack arg -> spilled  */
 210                                         var->regoff = parentargs_base + s1 + 1;
 211                                 }
 212                         }
 213
 214                 } else {                                     /* floating args         */
 215                         if (!md->params[p].inmemory) {           /* register arguments    */
 216                                 s2 = rd->argfltregs[s1];
 217                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 218                                         M_FLTMOVE(s2, var->regoff);
 219
 220                                 } else {                                         /* reg arg -> spilled    */
 221                                         M_DST(s2, REG_SP, var->regoff * 8);
 222                                 }
 223
 224                         } else {                                 /* stack arguments       */
 225                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 226                                         M_DLD(var->regoff, REG_SP, (parentargs_base + s1) * 8 + 8);
 227
 228                                 } else {
 229                                         var->regoff = parentargs_base + s1 + 1;
 230                                 }
 231                         }
 232                 }
 233         }  /* end for */
 234
 235         /* save monitorenter argument */
 236
 237 #if defined(USE_THREADS)
 238         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 239                 /* stack offset for monitor argument */
 240
 241                 s1 = rd->memuse;
 242
 243                 if (runverbose) {
 244                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 245
 246                         for (p = 0; p < INT_ARG_CNT; p++)
 247                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 248
 249                         for (p = 0; p < FLT_ARG_CNT; p++)
 250                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 251
 252                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 253                 }
 254
 255                 /* decide which monitor enter function to call */
 256
 257                 if (m->flags & ACC_STATIC) {
 258                         x86_64_mov_imm_reg(cd, (ptrint) m->class, REG_ITMP1);
 259                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, s1 * 8);
 260                         M_INTMOVE(REG_ITMP1, rd->argintregs[0]);
 261                         x86_64_mov_imm_reg(cd, (ptrint) BUILTIN_staticmonitorenter, REG_ITMP1);
 262                         x86_64_call_reg(cd, REG_ITMP1);
 263
 264                 } else {
 265                         x86_64_test_reg_reg(cd, rd->argintregs[0], rd->argintregs[0]);
 266                         x86_64_jcc(cd, X86_64_CC_Z, 0);
 267                         codegen_addxnullrefs(cd, cd->mcodeptr);
 268                         x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, s1 * 8);
 269                         x86_64_mov_imm_reg(cd, (ptrint) BUILTIN_monitorenter, REG_ITMP1);
 270                         x86_64_call_reg(cd, REG_ITMP1);
 271                 }
 272
 273                 if (runverbose) {
 274                         for (p = 0; p < INT_ARG_CNT; p++)
 275                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 276
 277                         for (p = 0; p < FLT_ARG_CNT; p++)
 278                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 279
 280                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 281                 }
 282         }
 283 #endif
 284
 285         /* Copy argument registers to stack and call trace function with pointer  */
 286         /* to arguments on stack.                                                 */
 287
 288         if (runverbose || opt_stat) {
 289                 M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 290
 291                 /* save integer argument registers */
 292
 293                 for (p = 0; p < INT_ARG_CNT; p++)
 294                         M_LST(rd->argintregs[p], REG_SP, (1 + p) * 8);
 295
 296                 /* save float argument registers */
 297
 298                 for (p = 0; p < FLT_ARG_CNT; p++)
 299                         M_DST(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 300
 301                 /* save temporary registers for leaf methods */
 302
 303                 if (m->isleafmethod) {
 304                         for (p = 0; p < INT_TMP_CNT; p++)
 305                                 M_LST(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 306
 307                         for (p = 0; p < FLT_TMP_CNT; p++)
 308                                 M_DST(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 309                 }
 310
 311                 if (runverbose) {
 312                         /* show integer hex code for float arguments */
 313
 314                         for (p = 0, l = 0; p < md->paramcount && p < INT_ARG_CNT; p++) {
 315                                 /* if the paramtype is a float, we have to right shift all    */
 316                                 /* following integer registers                                */
 317
 318                                 if (IS_FLT_DBL_TYPE(md->paramtypes[p].type)) {
 319                                         for (s1 = INT_ARG_CNT - 2; s1 >= p; s1--) {
 320                                                 M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
 321                                         }
 322
 323                                         x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 324                                         l++;
 325                                 }
 326                         }
 327
 328                         x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP2);
 329                         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_SP, 0 * 8);
 330                         x86_64_mov_imm_reg(cd, (ptrint) builtin_trace_args, REG_ITMP1);
 331                         x86_64_call_reg(cd, REG_ITMP1);
 332                 }
 333                 STATS({
 334                         x86_64_mov_imm_reg(cd, (ptrint) compiledinvokation, REG_ITMP1);
 335                         x86_64_call_reg(cd, REG_ITMP1);
 336                 })
 337
 338                 /* restore integer argument registers */
 339
 340                 for (p = 0; p < INT_ARG_CNT; p++)
 341                         M_LLD(rd->argintregs[p], REG_SP, (1 + p) * 8);
 342
 343                 /* restore float argument registers */
 344
 345                 for (p = 0; p < FLT_ARG_CNT; p++)
 346                         M_DLD(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 347
 348                 /* restore temporary registers for leaf methods */
 349
 350                 if (m->isleafmethod) {
 351                         for (p = 0; p < INT_TMP_CNT; p++)
 352                                 M_LLD(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 353
 354                         for (p = 0; p < FLT_TMP_CNT; p++)
 355                                 M_DLD(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 356                 }
 357
 358                 M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 359         }
 360
 361         }
 362
 363         /* end of header generation */
 364
 365         /* walk through all basic blocks */
 366         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 367
 368                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 369
 370                 if (bptr->flags >= BBREACHED) {
 371
 372                         /* branch resolving */
 373
 374                         branchref *bref;
 375                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 376                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 377                                                                   bref->branchpos,
 378                                                                   bptr->mpc);
 379                         }
 380
 381                 /* copy interface registers to their destination */
 382
 383                 src = bptr->instack;
 384                 len = bptr->indepth;
 385                 MCODECHECK(512);
 386
 387 #ifdef LSRA
 388                 if (opt_lsra) {
 389                         while (src != NULL) {
 390                                 len--;
 391                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 392                                         if (bptr->type == BBTYPE_SBR) {
 393                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 394                                                 if (!(src->flags & INMEMORY))
 395                                                         d= src->regoff;
 396                                                 else
 397                                                         d=REG_ITMP1;
 398                                                 x86_64_pop_reg(cd, d);
 399                                                 store_reg_to_var_int(src, d);
 400
 401                                         } else if (bptr->type == BBTYPE_EXH) {
 402                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 403                                                 if (!(src->flags & INMEMORY))
 404                                                         d= src->regoff;
 405                                                 else
 406                                                         d=REG_ITMP1;
 407                                                 M_INTMOVE(REG_ITMP1, d);
 408                                                 store_reg_to_var_int(src, d);
 409                                         }
 410                                 }
 411                                 src = src->prev;
 412                         }
 413
 414                 } else {
 415 #endif
 416
 417         while (src != NULL) {
 418                         len--;
 419                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 420                                 if (bptr->type == BBTYPE_SBR) {
 421                                         d = reg_of_var(rd, src, REG_ITMP1);
 422                                         x86_64_pop_reg(cd, d);
 423                                         store_reg_to_var_int(src, d);
 424
 425                                 } else if (bptr->type == BBTYPE_EXH) {
 426                                         d = reg_of_var(rd, src, REG_ITMP1);
 427                                         M_INTMOVE(REG_ITMP1, d);
 428                                         store_reg_to_var_int(src, d);
 429                                 }
 430
 431                         } else {
 432                                 d = reg_of_var(rd, src, REG_ITMP1);
 433                                 if ((src->varkind != STACKVAR)) {
 434                                         s2 = src->type;
 435                                         if (IS_FLT_DBL_TYPE(s2)) {
 436                                                 s1 = rd->interfaces[len][s2].regoff;
 437                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 438                                                         M_FLTMOVE(s1, d);
 439
 440                                                 } else {
 441                                                         x86_64_movq_membase_reg(cd, REG_SP, s1 * 8, d);
 442                                                 }
 443                                                 store_reg_to_var_flt(src, d);
 444
 445                                         } else {
 446                                                 s1 = rd->interfaces[len][s2].regoff;
 447                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 448                                                         M_INTMOVE(s1, d);
 449
 450                                                 } else {
 451                                                         x86_64_mov_membase_reg(cd, REG_SP, s1 * 8, d);
 452                                                 }
 453                                                 store_reg_to_var_int(src, d);
 454                                         }
 455                                 }
 456                         }
 457                         src = src->prev;
 458                 }
 459 #ifdef LSRA
 460                 }
 461 #endif
 462                 /* walk through all instructions */
 463
 464                 src = bptr->instack;
 465                 len = bptr->icount;
 466                 currentline=0;
 467                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 468                         if (iptr->line != currentline) {
 469                                 dseg_addlinenumber(cd, iptr->line, cd->mcodeptr);
 470                                 /*printf("%s : %d\n",m->name->text,iptr->line);*/
 471                                 currentline = iptr->line;
 472                         }
 473
 474                         MCODECHECK(1024);                         /* 1KB should be enough */
 475
 476                         switch (iptr->opc) {
 477                         case ICMD_INLINE_START: /* internal ICMDs                         */
 478                         case ICMD_INLINE_END:
 479                                 break;
 480
 481                         case ICMD_NOP:    /* ...  ==> ...                                 */
 482                                 break;
 483
 484                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 485                                 if (src->flags & INMEMORY) {
 486                                         x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
 487
 488                                 } else {
 489                                         x86_64_test_reg_reg(cd, src->regoff, src->regoff);
 490                                 }
 491                                 x86_64_jcc(cd, X86_64_CC_Z, 0);
 492                                 codegen_addxnullrefs(cd, cd->mcodeptr);
 493                                 break;
 494
 495                 /* constant operations ************************************************/
 496
 497                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 498                                       /* op1 = 0, val.i = constant                    */
 499
 500                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 501                         if (iptr->val.i == 0) {
 502                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 503                         } else {
 504                                 x86_64_movl_imm_reg(cd, iptr->val.i, d);
 505                         }
 506                         store_reg_to_var_int(iptr->dst, d);
 507                         break;
 508
 509                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 510                                       /* op1 = 0, val.a = constant                    */
 511
 512                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 513                         if (iptr->val.a == 0) {
 514                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 515                         } else {
 516                                 x86_64_mov_imm_reg(cd, (s8) iptr->val.a, d);
 517                         }
 518                         store_reg_to_var_int(iptr->dst, d);
 519                         break;
 520
 521                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 522                                       /* op1 = 0, val.l = constant                    */
 523
 524                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 525                         if (iptr->val.l == 0) {
 526                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 527                         } else {
 528                                 x86_64_mov_imm_reg(cd, iptr->val.l, d);
 529                         }
 530                         store_reg_to_var_int(iptr->dst, d);
 531                         break;
 532
 533                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 534                                       /* op1 = 0, val.f = constant                    */
 535
 536                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 537                         a = dseg_addfloat(cd, iptr->val.f);
 538                         x86_64_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + a, d);
 539                         store_reg_to_var_flt(iptr->dst, d);
 540                         break;
 541
 542                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 543                                       /* op1 = 0, val.d = constant                    */
 544
 545                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 546                         a = dseg_adddouble(cd, iptr->val.d);
 547                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, d);
 548                         store_reg_to_var_flt(iptr->dst, d);
 549                         break;
 550
 551
 552                 /* load/store operations **********************************************/
 553
 554                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 555                                       /* op1 = local variable                         */
 556
 557                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 558                         if ((iptr->dst->varkind == LOCALVAR) &&
 559                             (iptr->dst->varnum == iptr->op1)) {
 560                                 break;
 561                         }
 562                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 563                         if (var->flags & INMEMORY) {
 564                                 x86_64_movl_membase_reg(cd, REG_SP, var->regoff * 8, d);
 565                                 store_reg_to_var_int(iptr->dst, d);
 566
 567                         } else {
 568                                 if (iptr->dst->flags & INMEMORY) {
 569                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 570
 571                                 } else {
 572                                         M_INTMOVE(var->regoff, d);
 573                                 }
 574                         }
 575                         break;
 576
 577                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 578                 case ICMD_ALOAD:      /* op1 = local variable                         */
 579
 580                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 581                         if ((iptr->dst->varkind == LOCALVAR) &&
 582                             (iptr->dst->varnum == iptr->op1)) {
 583                                 break;
 584                         }
 585                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 586                         if (var->flags & INMEMORY) {
 587                                 x86_64_mov_membase_reg(cd, REG_SP, var->regoff * 8, d);
 588                                 store_reg_to_var_int(iptr->dst, d);
 589
 590                         } else {
 591                                 if (iptr->dst->flags & INMEMORY) {
 592                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 593
 594                                 } else {
 595                                         M_INTMOVE(var->regoff, d);
 596                                 }
 597                         }
 598                         break;
 599
 600                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 601                 case ICMD_DLOAD:      /* op1 = local variable                         */
 602
 603                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 604                         if ((iptr->dst->varkind == LOCALVAR) &&
 605                             (iptr->dst->varnum == iptr->op1)) {
 606                                 break;
 607                         }
 608                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 609                         if (var->flags & INMEMORY) {
 610                                 x86_64_movq_membase_reg(cd, REG_SP, var->regoff * 8, d);
 611                                 store_reg_to_var_flt(iptr->dst, d);
 612
 613                         } else {
 614                                 if (iptr->dst->flags & INMEMORY) {
 615                                         x86_64_movq_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 616
 617                                 } else {
 618                                         M_FLTMOVE(var->regoff, d);
 619                                 }
 620                         }
 621                         break;
 622
 623                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 624                 case ICMD_LSTORE:     /* op1 = local variable                         */
 625                 case ICMD_ASTORE:
 626
 627                         if ((src->varkind == LOCALVAR) &&
 628                             (src->varnum == iptr->op1)) {
 629                                 break;
 630                         }
 631                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 632                         if (var->flags & INMEMORY) {
 633                                 var_to_reg_int(s1, src, REG_ITMP1);
 634                                 x86_64_mov_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 635
 636                         } else {
 637                                 var_to_reg_int(s1, src, var->regoff);
 638                                 M_INTMOVE(s1, var->regoff);
 639                         }
 640                         break;
 641
 642                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 643                 case ICMD_DSTORE:     /* op1 = local variable                         */
 644
 645                         if ((src->varkind == LOCALVAR) &&
 646                             (src->varnum == iptr->op1)) {
 647                                 break;
 648                         }
 649                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 650                         if (var->flags & INMEMORY) {
 651                                 var_to_reg_flt(s1, src, REG_FTMP1);
 652                                 x86_64_movq_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 653
 654                         } else {
 655                                 var_to_reg_flt(s1, src, var->regoff);
 656                                 M_FLTMOVE(s1, var->regoff);
 657                         }
 658                         break;
 659
 660
 661                 /* pop/dup/swap operations ********************************************/
 662
 663                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 664
 665                 case ICMD_POP:        /* ..., value  ==> ...                          */
 666                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 667                         break;
 668
 669                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 670                         M_COPY(src, iptr->dst);
 671                         break;
 672
 673                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 674
 675                         M_COPY(src,       iptr->dst);
 676                         M_COPY(src->prev, iptr->dst->prev);
 677                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 678                         break;
 679
 680                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 681
 682                         M_COPY(src,             iptr->dst);
 683                         M_COPY(src->prev,       iptr->dst->prev);
 684                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 685                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 686                         break;
 687
 688                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 689
 690                         M_COPY(src,       iptr->dst);
 691                         M_COPY(src->prev, iptr->dst->prev);
 692                         break;
 693
 694                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 695
 696                         M_COPY(src,             iptr->dst);
 697                         M_COPY(src->prev,       iptr->dst->prev);
 698                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 699                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 700                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 701                         break;
 702
 703                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 704
 705                         M_COPY(src,                   iptr->dst);
 706                         M_COPY(src->prev,             iptr->dst->prev);
 707                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 708                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 709                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 710                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 711                         break;
 712
 713                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 714
 715                         M_COPY(src,       iptr->dst->prev);
 716                         M_COPY(src->prev, iptr->dst);
 717                         break;
 718
 719
 720                 /* integer operations *************************************************/
 721
 722                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 723
 724                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 725                         if (iptr->dst->flags & INMEMORY) {
 726                                 if (src->flags & INMEMORY) {
 727                                         if (src->regoff == iptr->dst->regoff) {
 728                                                 x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 729
 730                                         } else {
 731                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 732                                                 x86_64_negl_reg(cd, REG_ITMP1);
 733                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 734                                         }
 735
 736                                 } else {
 737                                         x86_64_movl_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 738                                         x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 739                                 }
 740
 741                         } else {
 742                                 if (src->flags & INMEMORY) {
 743                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 744                                         x86_64_negl_reg(cd, d);
 745
 746                                 } else {
 747                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 748                                         x86_64_negl_reg(cd, iptr->dst->regoff);
 749                                 }
 750                         }
 751                         break;
 752
 753                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 754
 755                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 756                         if (iptr->dst->flags & INMEMORY) {
 757                                 if (src->flags & INMEMORY) {
 758                                         if (src->regoff == iptr->dst->regoff) {
 759                                                 x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 760
 761                                         } else {
 762                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 763                                                 x86_64_neg_reg(cd, REG_ITMP1);
 764                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 765                                         }
 766
 767                                 } else {
 768                                         x86_64_mov_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 769                                         x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 770                                 }
 771
 772                         } else {
 773                                 if (src->flags & INMEMORY) {
 774                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 775                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 776
 777                                 } else {
 778                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 779                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 780                                 }
 781                         }
 782                         break;
 783
 784                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 785
 786                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 787                         if (src->flags & INMEMORY) {
 788                                 x86_64_movslq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 789
 790                         } else {
 791                                 x86_64_movslq_reg_reg(cd, src->regoff, d);
 792                         }
 793                         store_reg_to_var_int(iptr->dst, d);
 794                         break;
 795
 796                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 797
 798                         var_to_reg_int(s1, src, REG_ITMP1);
 799                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 800                         M_INTMOVE(s1, d);
 801                         store_reg_to_var_int(iptr->dst, d);
 802                         break;
 803
 804                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 805
 806                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 807                         if (src->flags & INMEMORY) {
 808                                 x86_64_movsbq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 809
 810                         } else {
 811                                 x86_64_movsbq_reg_reg(cd, src->regoff, d);
 812                         }
 813                         store_reg_to_var_int(iptr->dst, d);
 814                         break;
 815
 816                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 817
 818                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 819                         if (src->flags & INMEMORY) {
 820                                 x86_64_movzwq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 821
 822                         } else {
 823                                 x86_64_movzwq_reg_reg(cd, src->regoff, d);
 824                         }
 825                         store_reg_to_var_int(iptr->dst, d);
 826                         break;
 827
 828                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 829
 830                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 831                         if (src->flags & INMEMORY) {
 832                                 x86_64_movswq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 833
 834                         } else {
 835                                 x86_64_movswq_reg_reg(cd, src->regoff, d);
 836                         }
 837                         store_reg_to_var_int(iptr->dst, d);
 838                         break;
 839
 840
 841                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 842
 843                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 844                         x86_64_emit_ialu(cd, X86_64_ADD, src, iptr);
 845                         break;
 846
 847                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 848                                       /* val.i = constant                             */
 849
 850                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 851                         x86_64_emit_ialuconst(cd, X86_64_ADD, src, iptr);
 852                         break;
 853
 854                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 855
 856                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 857                         x86_64_emit_lalu(cd, X86_64_ADD, src, iptr);
 858                         break;
 859
 860                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 861                                       /* val.l = constant                             */
 862
 863                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 864                         x86_64_emit_laluconst(cd, X86_64_ADD, src, iptr);
 865                         break;
 866
 867                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 868
 869                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 870                         if (iptr->dst->flags & INMEMORY) {
 871                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 872                                         if (src->prev->regoff == iptr->dst->regoff) {
 873                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 874                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 875
 876                                         } else {
 877                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 878                                                 x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 879                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 880                                         }
 881
 882                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 883                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 884                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 885                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 886
 887                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 888                                         if (src->prev->regoff == iptr->dst->regoff) {
 889                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 890
 891                                         } else {
 892                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 893                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 894                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 895                                         }
 896
 897                                 } else {
 898                                         x86_64_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 899                                         x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 900                                 }
 901
 902                         } else {
 903                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 904                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 905                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 906
 907                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 908                                         M_INTMOVE(src->prev->regoff, d);
 909                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 910
 911                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 912                                         /* workaround for reg alloc */
 913                                         if (src->regoff == iptr->dst->regoff) {
 914                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 915                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 916                                                 M_INTMOVE(REG_ITMP1, d);
 917
 918                                         } else {
 919                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 920                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 921                                         }
 922
 923                                 } else {
 924                                         /* workaround for reg alloc */
 925                                         if (src->regoff == iptr->dst->regoff) {
 926                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
 927                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 928                                                 M_INTMOVE(REG_ITMP1, d);
 929
 930                                         } else {
 931                                                 M_INTMOVE(src->prev->regoff, d);
 932                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 933                                         }
 934                                 }
 935                         }
 936                         break;
 937
 938                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 939                                       /* val.i = constant                             */
 940
 941                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 942                         x86_64_emit_ialuconst(cd, X86_64_SUB, src, iptr);
 943                         break;
 944
 945                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 946
 947                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 948                         if (iptr->dst->flags & INMEMORY) {
 949                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 950                                         if (src->prev->regoff == iptr->dst->regoff) {
 951                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 952                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 953
 954                                         } else {
 955                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 956                                                 x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 957                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 958                                         }
 959
 960                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 961                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 962                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 963                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 964
 965                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 966                                         if (src->prev->regoff == iptr->dst->regoff) {
 967                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 968
 969                                         } else {
 970                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 971                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 972                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 973                                         }
 974
 975                                 } else {
 976                                         x86_64_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 977                                         x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 978                                 }
 979
 980                         } else {
 981                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 982                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 983                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 984
 985                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 986                                         M_INTMOVE(src->prev->regoff, d);
 987                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 988
 989                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 990                                         /* workaround for reg alloc */
 991                                         if (src->regoff == iptr->dst->regoff) {
 992                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 993                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 994                                                 M_INTMOVE(REG_ITMP1, d);
 995
 996                                         } else {
 997                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 998                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
 999                                         }
1000
1001                                 } else {
1002                                         /* workaround for reg alloc */
1003                                         if (src->regoff == iptr->dst->regoff) {
1004                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1005                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1006                                                 M_INTMOVE(REG_ITMP1, d);
1007
1008                                         } else {
1009                                                 M_INTMOVE(src->prev->regoff, d);
1010                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1011                                         }
1012                                 }
1013                         }
1014                         break;
1015
1016                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1017                                       /* val.l = constant                             */
1018
1019                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1020                         x86_64_emit_laluconst(cd, X86_64_SUB, src, iptr);
1021                         break;
1022
1023                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1024
1025                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1026                         if (iptr->dst->flags & INMEMORY) {
1027                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1028                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1029                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1030                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1031
1032                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1033                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1034                                         x86_64_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1035                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1036
1037                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1038                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1039                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1040                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1041
1042                                 } else {
1043                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1044                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1045                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1046                                 }
1047
1048                         } else {
1049                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1050                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1051                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1052
1053                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1054                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1055                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1056
1057                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1058                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1059                                         x86_64_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1060
1061                                 } else {
1062                                         if (src->regoff == iptr->dst->regoff) {
1063                                                 x86_64_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1064
1065                                         } else {
1066                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1067                                                 x86_64_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1068                                         }
1069                                 }
1070                         }
1071                         break;
1072
1073                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1074                                       /* val.i = constant                             */
1075
1076                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1077                         if (iptr->dst->flags & INMEMORY) {
1078                                 if (src->flags & INMEMORY) {
1079                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1080                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1081
1082                                 } else {
1083                                         x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1084                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1085                                 }
1086
1087                         } else {
1088                                 if (src->flags & INMEMORY) {
1089                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1090
1091                                 } else {
1092                                         if (iptr->val.i == 2) {
1093                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1094                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1095
1096                                         } else {
1097                                                 x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1098                                         }
1099                                 }
1100                         }
1101                         break;
1102
1103                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1104
1105                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1106                         if (iptr->dst->flags & INMEMORY) {
1107                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1108                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1109                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1110                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1111
1112                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1113                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1114                                         x86_64_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1115                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1116
1117                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1118                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1119                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1120                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1121
1122                                 } else {
1123                                         x86_64_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1124                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1125                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1126                                 }
1127
1128                         } else {
1129                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1130                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1131                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1132
1133                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1134                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1135                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1136
1137                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1138                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1139                                         x86_64_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1140
1141                                 } else {
1142                                         if (src->regoff == iptr->dst->regoff) {
1143                                                 x86_64_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1144
1145                                         } else {
1146                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1147                                                 x86_64_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1148                                         }
1149                                 }
1150                         }
1151                         break;
1152
1153                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1154                                       /* val.l = constant                             */
1155
1156                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1157                         if (iptr->dst->flags & INMEMORY) {
1158                                 if (src->flags & INMEMORY) {
1159                                         if (IS_IMM32(iptr->val.l)) {
1160                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1161
1162                                         } else {
1163                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1164                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1165                                         }
1166                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1167
1168                                 } else {
1169                                         if (IS_IMM32(iptr->val.l)) {
1170                                                 x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1171
1172                                         } else {
1173                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1174                                                 x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1175                                         }
1176                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1177                                 }
1178
1179                         } else {
1180                                 if (src->flags & INMEMORY) {
1181                                         if (IS_IMM32(iptr->val.l)) {
1182                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1183
1184                                         } else {
1185                                                 x86_64_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1186                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1187                                         }
1188
1189                                 } else {
1190                                         /* should match in many cases */
1191                                         if (iptr->val.l == 2) {
1192                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1193                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1194
1195                                         } else {
1196                                                 if (IS_IMM32(iptr->val.l)) {
1197                                                         x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1198
1199                                                 } else {
1200                                                         x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1201                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1202                                                         x86_64_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1203                                                 }
1204                                         }
1205                                 }
1206                         }
1207                         break;
1208
1209                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1210
1211                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1212                 if (src->prev->flags & INMEMORY) {
1213                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1214
1215                         } else {
1216                                 M_INTMOVE(src->prev->regoff, RAX);
1217                         }
1218
1219                         if (src->flags & INMEMORY) {
1220                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1221
1222                         } else {
1223                                 M_INTMOVE(src->regoff, REG_ITMP3);
1224                         }
1225                         gen_div_check(src);
1226
1227                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX);    /* check as described in jvm spec */
1228                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1229                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);      /* 4 bytes */
1230                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                  /* 6 bytes */
1231
1232                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1233                         x86_64_cltd(cd);
1234                         x86_64_idivl_reg(cd, REG_ITMP3);
1235
1236                         if (iptr->dst->flags & INMEMORY) {
1237                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1238                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1239
1240                         } else {
1241                                 M_INTMOVE(RAX, iptr->dst->regoff);
1242
1243                                 if (iptr->dst->regoff != RDX) {
1244                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1245                                 }
1246                         }
1247                         break;
1248
1249                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1250                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1251                         if (src->prev->flags & INMEMORY) {
1252                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1253
1254                         } else {
1255                                 M_INTMOVE(src->prev->regoff, RAX);
1256                         }
1257
1258                         if (src->flags & INMEMORY) {
1259                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1260
1261                         } else {
1262                                 M_INTMOVE(src->regoff, REG_ITMP3);
1263                         }
1264                         gen_div_check(src);
1265
1266                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1267
1268                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX);    /* check as described in jvm spec */
1269                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1270
1271
1272                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);           /* 2 bytes */
1273                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);      /* 4 bytes */
1274                         x86_64_jcc(cd, X86_64_CC_E, 1 + 3);                      /* 6 bytes */
1275
1276                         x86_64_cltd(cd);
1277                         x86_64_idivl_reg(cd, REG_ITMP3);
1278
1279                         if (iptr->dst->flags & INMEMORY) {
1280                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1281                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1282
1283                         } else {
1284                                 M_INTMOVE(RDX, iptr->dst->regoff);
1285
1286                                 if (iptr->dst->regoff != RDX) {
1287                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1288                                 }
1289                         }
1290                         break;
1291
1292                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1293                                       /* val.i = constant                             */
1294
1295                         var_to_reg_int(s1, src, REG_ITMP1);
1296                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1297                         M_INTMOVE(s1, REG_ITMP1);
1298                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1299                         x86_64_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1300                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1301                         x86_64_shiftl_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1302                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1303                         store_reg_to_var_int(iptr->dst, d);
1304                         break;
1305
1306                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1307                                       /* val.i = constant                             */
1308
1309                         var_to_reg_int(s1, src, REG_ITMP1);
1310                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1311                         M_INTMOVE(s1, REG_ITMP1);
1312                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1313                         x86_64_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1314                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1315                         x86_64_alul_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1316                         x86_64_alul_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1317                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1318                         store_reg_to_var_int(iptr->dst, d);
1319                         break;
1320
1321
1322                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1323
1324                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1325                 if (src->prev->flags & INMEMORY) {
1326                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1327
1328                         } else {
1329                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1330                         }
1331
1332                         if (src->flags & INMEMORY) {
1333                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1334
1335                         } else {
1336                                 M_INTMOVE(src->regoff, REG_ITMP3);
1337                         }
1338                         gen_div_check(src);
1339
1340                         x86_64_mov_imm_reg(cd, 0x8000000000000000LL, REG_ITMP2);    /* check as described in jvm spec */
1341                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
1342                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1343                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);          /* 4 bytes */
1344                         x86_64_jcc(cd, X86_64_CC_E, 3 + 2 + 3);                     /* 6 bytes */
1345
1346                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1347                         x86_64_cqto(cd);
1348                         x86_64_idiv_reg(cd, REG_ITMP3);
1349
1350                         if (iptr->dst->flags & INMEMORY) {
1351                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1352                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1353
1354                         } else {
1355                                 M_INTMOVE(RAX, iptr->dst->regoff);
1356
1357                                 if (iptr->dst->regoff != RDX) {
1358                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1359                                 }
1360                         }
1361                         break;
1362
1363                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1364
1365                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1366                         if (src->prev->flags & INMEMORY) {
1367                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1368
1369                         } else {
1370                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1371                         }
1372
1373                         if (src->flags & INMEMORY) {
1374                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1375
1376                         } else {
1377                                 M_INTMOVE(src->regoff, REG_ITMP3);
1378                         }
1379                         gen_div_check(src);
1380
1381                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1382
1383                         x86_64_mov_imm_reg(cd, 0x8000000000000000LL, REG_ITMP2);    /* check as described in jvm spec */
1384                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
1385                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1386
1387
1388                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);              /* 2 bytes */
1389                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);          /* 4 bytes */
1390                         x86_64_jcc(cd, X86_64_CC_E, 2 + 3);                         /* 6 bytes */
1391
1392                         x86_64_cqto(cd);
1393                         x86_64_idiv_reg(cd, REG_ITMP3);
1394
1395                         if (iptr->dst->flags & INMEMORY) {
1396                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1397                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1398
1399                         } else {
1400                                 M_INTMOVE(RDX, iptr->dst->regoff);
1401
1402                                 if (iptr->dst->regoff != RDX) {
1403                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1404                                 }
1405                         }
1406                         break;
1407
1408                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1409                                       /* val.i = constant                             */
1410
1411                         var_to_reg_int(s1, src, REG_ITMP1);
1412                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1413                         M_INTMOVE(s1, REG_ITMP1);
1414                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1415                         x86_64_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1416                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1417                         x86_64_shift_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1418                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1419                         store_reg_to_var_int(iptr->dst, d);
1420                         break;
1421
1422                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1423                                       /* val.l = constant                             */
1424
1425                         var_to_reg_int(s1, src, REG_ITMP1);
1426                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1427                         M_INTMOVE(s1, REG_ITMP1);
1428                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1429                         x86_64_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1430                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1431                         x86_64_alu_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1432                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1433                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1434                         store_reg_to_var_int(iptr->dst, d);
1435                         break;
1436
1437                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1438
1439                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1440                         x86_64_emit_ishift(cd, X86_64_SHL, src, iptr);
1441                         break;
1442
1443                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1444                                       /* val.i = constant                             */
1445
1446                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1447                         x86_64_emit_ishiftconst(cd, X86_64_SHL, src, iptr);
1448                         break;
1449
1450                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1451
1452                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1453                         x86_64_emit_ishift(cd, X86_64_SAR, src, iptr);
1454                         break;
1455
1456                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1457                                       /* val.i = constant                             */
1458
1459                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1460                         x86_64_emit_ishiftconst(cd, X86_64_SAR, src, iptr);
1461                         break;
1462
1463                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1464
1465                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1466                         x86_64_emit_ishift(cd, X86_64_SHR, src, iptr);
1467                         break;
1468
1469                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1470                                       /* val.i = constant                             */
1471
1472                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1473                         x86_64_emit_ishiftconst(cd, X86_64_SHR, src, iptr);
1474                         break;
1475
1476                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1477
1478                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1479                         x86_64_emit_lshift(cd, X86_64_SHL, src, iptr);
1480                         break;
1481
1482         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1483                                           /* val.i = constant                             */
1484
1485                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1486                         x86_64_emit_lshiftconst(cd, X86_64_SHL, src, iptr);
1487                         break;
1488
1489                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1490
1491                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1492                         x86_64_emit_lshift(cd, X86_64_SAR, src, iptr);
1493                         break;
1494
1495                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1496                                       /* val.i = constant                             */
1497
1498                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1499                         x86_64_emit_lshiftconst(cd, X86_64_SAR, src, iptr);
1500                         break;
1501
1502                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1503
1504                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1505                         x86_64_emit_lshift(cd, X86_64_SHR, src, iptr);
1506                         break;
1507
1508                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1509                                       /* val.l = constant                             */
1510
1511                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1512                         x86_64_emit_lshiftconst(cd, X86_64_SHR, src, iptr);
1513                         break;
1514
1515                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1516
1517                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1518                         x86_64_emit_ialu(cd, X86_64_AND, src, iptr);
1519                         break;
1520
1521                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1522                                       /* val.i = constant                             */
1523
1524                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1525                         x86_64_emit_ialuconst(cd, X86_64_AND, src, iptr);
1526                         break;
1527
1528                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1529
1530                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1531                         x86_64_emit_lalu(cd, X86_64_AND, src, iptr);
1532                         break;
1533
1534                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1535                                       /* val.l = constant                             */
1536
1537                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1538                         x86_64_emit_laluconst(cd, X86_64_AND, src, iptr);
1539                         break;
1540
1541                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1542
1543                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1544                         x86_64_emit_ialu(cd, X86_64_OR, src, iptr);
1545                         break;
1546
1547                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1548                                       /* val.i = constant                             */
1549
1550                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1551                         x86_64_emit_ialuconst(cd, X86_64_OR, src, iptr);
1552                         break;
1553
1554                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1555
1556                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1557                         x86_64_emit_lalu(cd, X86_64_OR, src, iptr);
1558                         break;
1559
1560                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1561                                       /* val.l = constant                             */
1562
1563                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1564                         x86_64_emit_laluconst(cd, X86_64_OR, src, iptr);
1565                         break;
1566
1567                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1568
1569                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1570                         x86_64_emit_ialu(cd, X86_64_XOR, src, iptr);
1571                         break;
1572
1573                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1574                                       /* val.i = constant                             */
1575
1576                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1577                         x86_64_emit_ialuconst(cd, X86_64_XOR, src, iptr);
1578                         break;
1579
1580                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1581
1582                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1583                         x86_64_emit_lalu(cd, X86_64_XOR, src, iptr);
1584                         break;
1585
1586                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1587                                       /* val.l = constant                             */
1588
1589                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1590                         x86_64_emit_laluconst(cd, X86_64_XOR, src, iptr);
1591                         break;
1592
1593
1594                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1595                                       /* op1 = variable, val.i = constant             */
1596
1597                         /* using inc and dec is definitely faster than add -- tested      */
1598                         /* with sieve                                                     */
1599
1600                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1601                         d = var->regoff;
1602                         if (var->flags & INMEMORY) {
1603                                 if (iptr->val.i == 1) {
1604                                         x86_64_incl_membase(cd, REG_SP, d * 8);
1605
1606                                 } else if (iptr->val.i == -1) {
1607                                         x86_64_decl_membase(cd, REG_SP, d * 8);
1608
1609                                 } else {
1610                                         x86_64_alul_imm_membase(cd, X86_64_ADD, iptr->val.i, REG_SP, d * 8);
1611                                 }
1612
1613                         } else {
1614                                 if (iptr->val.i == 1) {
1615                                         x86_64_incl_reg(cd, d);
1616
1617                                 } else if (iptr->val.i == -1) {
1618                                         x86_64_decl_reg(cd, d);
1619
1620                                 } else {
1621                                         x86_64_alul_imm_reg(cd, X86_64_ADD, iptr->val.i, d);
1622                                 }
1623                         }
1624                         break;
1625
1626
1627                 /* floating operations ************************************************/
1628
1629                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1630
1631                         var_to_reg_flt(s1, src, REG_FTMP1);
1632                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1633                         a = dseg_adds4(cd, 0x80000000);
1634                         M_FLTMOVE(s1, d);
1635                         x86_64_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, REG_FTMP2);
1636                         x86_64_xorps_reg_reg(cd, REG_FTMP2, d);
1637                         store_reg_to_var_flt(iptr->dst, d);
1638                         break;
1639
1640                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1641
1642                         var_to_reg_flt(s1, src, REG_FTMP1);
1643                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1644                         a = dseg_adds8(cd, 0x8000000000000000);
1645                         M_FLTMOVE(s1, d);
1646                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, REG_FTMP2);
1647                         x86_64_xorpd_reg_reg(cd, REG_FTMP2, d);
1648                         store_reg_to_var_flt(iptr->dst, d);
1649                         break;
1650
1651                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1652
1653                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1654                         var_to_reg_flt(s2, src, REG_FTMP2);
1655                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1656                         if (s1 == d) {
1657                                 x86_64_addss_reg_reg(cd, s2, d);
1658                         } else if (s2 == d) {
1659                                 x86_64_addss_reg_reg(cd, s1, d);
1660                         } else {
1661                                 M_FLTMOVE(s1, d);
1662                                 x86_64_addss_reg_reg(cd, s2, d);
1663                         }
1664                         store_reg_to_var_flt(iptr->dst, d);
1665                         break;
1666
1667                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1668
1669                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1670                         var_to_reg_flt(s2, src, REG_FTMP2);
1671                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1672                         if (s1 == d) {
1673                                 x86_64_addsd_reg_reg(cd, s2, d);
1674                         } else if (s2 == d) {
1675                                 x86_64_addsd_reg_reg(cd, s1, d);
1676                         } else {
1677                                 M_FLTMOVE(s1, d);
1678                                 x86_64_addsd_reg_reg(cd, s2, d);
1679                         }
1680                         store_reg_to_var_flt(iptr->dst, d);
1681                         break;
1682
1683                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1684
1685                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1686                         var_to_reg_flt(s2, src, REG_FTMP2);
1687                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1688                         if (s2 == d) {
1689                                 M_FLTMOVE(s2, REG_FTMP2);
1690                                 s2 = REG_FTMP2;
1691                         }
1692                         M_FLTMOVE(s1, d);
1693                         x86_64_subss_reg_reg(cd, s2, d);
1694                         store_reg_to_var_flt(iptr->dst, d);
1695                         break;
1696
1697                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1698
1699                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1700                         var_to_reg_flt(s2, src, REG_FTMP2);
1701                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1702                         if (s2 == d) {
1703                                 M_FLTMOVE(s2, REG_FTMP2);
1704                                 s2 = REG_FTMP2;
1705                         }
1706                         M_FLTMOVE(s1, d);
1707                         x86_64_subsd_reg_reg(cd, s2, d);
1708                         store_reg_to_var_flt(iptr->dst, d);
1709                         break;
1710
1711                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1712
1713                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1714                         var_to_reg_flt(s2, src, REG_FTMP2);
1715                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1716                         if (s1 == d) {
1717                                 x86_64_mulss_reg_reg(cd, s2, d);
1718                         } else if (s2 == d) {
1719                                 x86_64_mulss_reg_reg(cd, s1, d);
1720                         } else {
1721                                 M_FLTMOVE(s1, d);
1722                                 x86_64_mulss_reg_reg(cd, s2, d);
1723                         }
1724                         store_reg_to_var_flt(iptr->dst, d);
1725                         break;
1726
1727                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1728
1729                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1730                         var_to_reg_flt(s2, src, REG_FTMP2);
1731                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1732                         if (s1 == d) {
1733                                 x86_64_mulsd_reg_reg(cd, s2, d);
1734                         } else if (s2 == d) {
1735                                 x86_64_mulsd_reg_reg(cd, s1, d);
1736                         } else {
1737                                 M_FLTMOVE(s1, d);
1738                                 x86_64_mulsd_reg_reg(cd, s2, d);
1739                         }
1740                         store_reg_to_var_flt(iptr->dst, d);
1741                         break;
1742
1743                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1744
1745                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1746                         var_to_reg_flt(s2, src, REG_FTMP2);
1747                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1748                         if (s2 == d) {
1749                                 M_FLTMOVE(s2, REG_FTMP2);
1750                                 s2 = REG_FTMP2;
1751                         }
1752                         M_FLTMOVE(s1, d);
1753                         x86_64_divss_reg_reg(cd, s2, d);
1754                         store_reg_to_var_flt(iptr->dst, d);
1755                         break;
1756
1757                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1758
1759                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1760                         var_to_reg_flt(s2, src, REG_FTMP2);
1761                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1762                         if (s2 == d) {
1763                                 M_FLTMOVE(s2, REG_FTMP2);
1764                                 s2 = REG_FTMP2;
1765                         }
1766                         M_FLTMOVE(s1, d);
1767                         x86_64_divsd_reg_reg(cd, s2, d);
1768                         store_reg_to_var_flt(iptr->dst, d);
1769                         break;
1770
1771                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1772
1773                         var_to_reg_int(s1, src, REG_ITMP1);
1774                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1775                         x86_64_cvtsi2ss_reg_reg(cd, s1, d);
1776                         store_reg_to_var_flt(iptr->dst, d);
1777                         break;
1778
1779                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1780
1781                         var_to_reg_int(s1, src, REG_ITMP1);
1782                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1783                         x86_64_cvtsi2sd_reg_reg(cd, s1, d);
1784                         store_reg_to_var_flt(iptr->dst, d);
1785                         break;
1786
1787                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1788
1789                         var_to_reg_int(s1, src, REG_ITMP1);
1790                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1791                         x86_64_cvtsi2ssq_reg_reg(cd, s1, d);
1792                         store_reg_to_var_flt(iptr->dst, d);
1793                         break;
1794
1795                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1796
1797                         var_to_reg_int(s1, src, REG_ITMP1);
1798                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1799                         x86_64_cvtsi2sdq_reg_reg(cd, s1, d);
1800                         store_reg_to_var_flt(iptr->dst, d);
1801                         break;
1802
1803                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1804
1805                         var_to_reg_flt(s1, src, REG_FTMP1);
1806                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1807                         x86_64_cvttss2si_reg_reg(cd, s1, d);
1808                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1809                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1810                         x86_64_jcc(cd, X86_64_CC_NE, a);
1811                         M_FLTMOVE(s1, REG_FTMP1);
1812                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2i, REG_ITMP2);
1813                         x86_64_call_reg(cd, REG_ITMP2);
1814                         M_INTMOVE(REG_RESULT, d);
1815                         store_reg_to_var_int(iptr->dst, d);
1816                         break;
1817
1818                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1819
1820                         var_to_reg_flt(s1, src, REG_FTMP1);
1821                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1822                         x86_64_cvttsd2si_reg_reg(cd, s1, d);
1823                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1824                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1825                         x86_64_jcc(cd, X86_64_CC_NE, a);
1826                         M_FLTMOVE(s1, REG_FTMP1);
1827                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2i, REG_ITMP2);
1828                         x86_64_call_reg(cd, REG_ITMP2);
1829                         M_INTMOVE(REG_RESULT, d);
1830                         store_reg_to_var_int(iptr->dst, d);
1831                         break;
1832
1833                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1834
1835                         var_to_reg_flt(s1, src, REG_FTMP1);
1836                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1837                         x86_64_cvttss2siq_reg_reg(cd, s1, d);
1838                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1839                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1840                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1841                         x86_64_jcc(cd, X86_64_CC_NE, a);
1842                         M_FLTMOVE(s1, REG_FTMP1);
1843                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2l, REG_ITMP2);
1844                         x86_64_call_reg(cd, REG_ITMP2);
1845                         M_INTMOVE(REG_RESULT, d);
1846                         store_reg_to_var_int(iptr->dst, d);
1847                         break;
1848
1849                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1850
1851                         var_to_reg_flt(s1, src, REG_FTMP1);
1852                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1853                         x86_64_cvttsd2siq_reg_reg(cd, s1, d);
1854                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1855                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1856                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1857                         x86_64_jcc(cd, X86_64_CC_NE, a);
1858                         M_FLTMOVE(s1, REG_FTMP1);
1859                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2l, REG_ITMP2);
1860                         x86_64_call_reg(cd, REG_ITMP2);
1861                         M_INTMOVE(REG_RESULT, d);
1862                         store_reg_to_var_int(iptr->dst, d);
1863                         break;
1864
1865                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1866
1867                         var_to_reg_flt(s1, src, REG_FTMP1);
1868                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1869                         x86_64_cvtss2sd_reg_reg(cd, s1, d);
1870                         store_reg_to_var_flt(iptr->dst, d);
1871                         break;
1872
1873                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1874
1875                         var_to_reg_flt(s1, src, REG_FTMP1);
1876                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1877                         x86_64_cvtsd2ss_reg_reg(cd, s1, d);
1878                         store_reg_to_var_flt(iptr->dst, d);
1879                         break;
1880
1881                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1882                                           /* == => 0, < => 1, > => -1 */
1883
1884                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1885                         var_to_reg_flt(s2, src, REG_FTMP2);
1886                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1887                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1888                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1889                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1890                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1891                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1892                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1893                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1894                         store_reg_to_var_int(iptr->dst, d);
1895                         break;
1896
1897                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1898                                           /* == => 0, < => 1, > => -1 */
1899
1900                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1901                         var_to_reg_flt(s2, src, REG_FTMP2);
1902                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1903                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1904                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1905                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1906                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1907                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1908                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1909                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1910                         store_reg_to_var_int(iptr->dst, d);
1911                         break;
1912
1913                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1914                                           /* == => 0, < => 1, > => -1 */
1915
1916                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1917                         var_to_reg_flt(s2, src, REG_FTMP2);
1918                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1919                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1920                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1921                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1922                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1923                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1924                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1925                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1926                         store_reg_to_var_int(iptr->dst, d);
1927                         break;
1928
1929                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1930                                           /* == => 0, < => 1, > => -1 */
1931
1932                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1933                         var_to_reg_flt(s2, src, REG_FTMP2);
1934                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1935                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1936                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1937                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1938                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1939                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1940                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1941                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1942                         store_reg_to_var_int(iptr->dst, d);
1943                         break;
1944
1945
1946                 /* memory operations **************************************************/
1947
1948                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1949
1950                         var_to_reg_int(s1, src, REG_ITMP1);
1951                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1952                         gen_nullptr_check(s1);
1953                         x86_64_movl_membase_reg(cd, s1, OFFSET(java_arrayheader, size), d);
1954                         store_reg_to_var_int(iptr->dst, d);
1955                         break;
1956
1957                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1958
1959                         var_to_reg_int(s1, src->prev, REG_ITMP1);
1960                         var_to_reg_int(s2, src, REG_ITMP2);
1961                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1962                         if (iptr->op1 == 0) {
1963                                 gen_nullptr_check(s1);
1964                                 gen_bound_check;
1965                         }
1966                         x86_64_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1967                         store_reg_to_var_int(iptr->dst, d);
1968                         break;
1969
1970                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1971
1972                         var_to_reg_int(s1, src->prev, REG_ITMP1);
1973                         var_to_reg_int(s2, src, REG_ITMP2);
1974                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1975                         if (iptr->op1 == 0) {
1976                                 gen_nullptr_check(s1);
1977                                 gen_bound_check;
1978                         }
1979                         x86_64_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1980                         store_reg_to_var_int(iptr->dst, d);
1981                         break;
1982
1983                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1984
1985                         var_to_reg_int(s1, src->prev, REG_ITMP1);
1986                         var_to_reg_int(s2, src, REG_ITMP2);
1987                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1988                         if (iptr->op1 == 0) {
1989                                 gen_nullptr_check(s1);
1990                                 gen_bound_check;
1991                         }
1992                         x86_64_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1993                         store_reg_to_var_int(iptr->dst, d);
1994                         break;
1995
1996                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1997
1998                         var_to_reg_int(s1, src->prev, REG_ITMP1);
1999                         var_to_reg_int(s2, src, REG_ITMP2);
2000                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2001                         if (iptr->op1 == 0) {
2002                                 gen_nullptr_check(s1);
2003                                 gen_bound_check;
2004                         }
2005                         x86_64_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2006                         store_reg_to_var_flt(iptr->dst, d);
2007                         break;
2008
2009                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2010
2011                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2012                         var_to_reg_int(s2, src, REG_ITMP2);
2013                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2014                         if (iptr->op1 == 0) {
2015                                 gen_nullptr_check(s1);
2016                                 gen_bound_check;
2017                         }
2018                         x86_64_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2019                         store_reg_to_var_flt(iptr->dst, d);
2020                         break;
2021
2022                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2023
2024                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2025                         var_to_reg_int(s2, src, REG_ITMP2);
2026                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2027                         if (iptr->op1 == 0) {
2028                                 gen_nullptr_check(s1);
2029                                 gen_bound_check;
2030                         }
2031                         x86_64_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2032                         store_reg_to_var_int(iptr->dst, d);
2033                         break;
2034
2035                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2036
2037                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2038                         var_to_reg_int(s2, src, REG_ITMP2);
2039                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2040                         if (iptr->op1 == 0) {
2041                                 gen_nullptr_check(s1);
2042                                 gen_bound_check;
2043                         }
2044                         x86_64_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2045                         store_reg_to_var_int(iptr->dst, d);
2046                         break;
2047
2048                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2049
2050                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2051                         var_to_reg_int(s2, src, REG_ITMP2);
2052                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2053                         if (iptr->op1 == 0) {
2054                                 gen_nullptr_check(s1);
2055                                 gen_bound_check;
2056                         }
2057                         x86_64_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2058                         store_reg_to_var_int(iptr->dst, d);
2059                         break;
2060
2061
2062                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2063
2064                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2065                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2066                         if (iptr->op1 == 0) {
2067                                 gen_nullptr_check(s1);
2068                                 gen_bound_check;
2069                         }
2070                         var_to_reg_int(s3, src, REG_ITMP3);
2071                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2072                         break;
2073
2074                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2075
2076                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2077                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2078                         if (iptr->op1 == 0) {
2079                                 gen_nullptr_check(s1);
2080                                 gen_bound_check;
2081                         }
2082                         var_to_reg_int(s3, src, REG_ITMP3);
2083                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2084                         break;
2085
2086                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2087
2088                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2089                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2090                         if (iptr->op1 == 0) {
2091                                 gen_nullptr_check(s1);
2092                                 gen_bound_check;
2093                         }
2094                         var_to_reg_int(s3, src, REG_ITMP3);
2095                         x86_64_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2096                         break;
2097
2098                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2099
2100                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2101                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2102                         if (iptr->op1 == 0) {
2103                                 gen_nullptr_check(s1);
2104                                 gen_bound_check;
2105                         }
2106                         var_to_reg_flt(s3, src, REG_FTMP3);
2107                         x86_64_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2108                         break;
2109
2110                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2111
2112                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2113                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2114                         if (iptr->op1 == 0) {
2115                                 gen_nullptr_check(s1);
2116                                 gen_bound_check;
2117                         }
2118                         var_to_reg_flt(s3, src, REG_FTMP3);
2119                         x86_64_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2120                         break;
2121
2122                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2123
2124                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2125                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2126                         if (iptr->op1 == 0) {
2127                                 gen_nullptr_check(s1);
2128                                 gen_bound_check;
2129                         }
2130                         var_to_reg_int(s3, src, REG_ITMP3);
2131                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2132                         break;
2133
2134                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2135
2136                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2137                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2138                         if (iptr->op1 == 0) {
2139                                 gen_nullptr_check(s1);
2140                                 gen_bound_check;
2141                         }
2142                         var_to_reg_int(s3, src, REG_ITMP3);
2143                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2144                         break;
2145
2146                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2147
2148                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2149                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2150                         if (iptr->op1 == 0) {
2151                                 gen_nullptr_check(s1);
2152                                 gen_bound_check;
2153                         }
2154                         var_to_reg_int(s3, src, REG_ITMP3);
2155                         x86_64_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2156                         break;
2157
2158                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2159
2160                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2161                         var_to_reg_int(s2, src, REG_ITMP2);
2162                         if (iptr->op1 == 0) {
2163                                 gen_nullptr_check(s1);
2164                                 gen_bound_check;
2165                         }
2166                         x86_64_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2167                         break;
2168
2169                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2170
2171                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2172                         var_to_reg_int(s2, src, REG_ITMP2);
2173                         if (iptr->op1 == 0) {
2174                                 gen_nullptr_check(s1);
2175                                 gen_bound_check;
2176                         }
2177
2178                         if (IS_IMM32(iptr->val.l)) {
2179                                 x86_64_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2180
2181                         } else {
2182                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2183                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2184                         }
2185                         break;
2186
2187                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2188
2189                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2190                         var_to_reg_int(s2, src, REG_ITMP2);
2191                         if (iptr->op1 == 0) {
2192                                 gen_nullptr_check(s1);
2193                                 gen_bound_check;
2194                         }
2195                         x86_64_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2196                         break;
2197
2198                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2199
2200                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2201                         var_to_reg_int(s2, src, REG_ITMP2);
2202                         if (iptr->op1 == 0) {
2203                                 gen_nullptr_check(s1);
2204                                 gen_bound_check;
2205                         }
2206                         x86_64_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2207                         break;
2208
2209                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2210
2211                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2212                         var_to_reg_int(s2, src, REG_ITMP2);
2213                         if (iptr->op1 == 0) {
2214                                 gen_nullptr_check(s1);
2215                                 gen_bound_check;
2216                         }
2217                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2218                         break;
2219
2220                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2221
2222                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2223                         var_to_reg_int(s2, src, REG_ITMP2);
2224                         if (iptr->op1 == 0) {
2225                                 gen_nullptr_check(s1);
2226                                 gen_bound_check;
2227                         }
2228                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2229                         break;
2230
2231
2232                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2233                                       /* op1 = type, val.a = field address            */
2234
2235                         if (!iptr->val.a) {
2236                                 codegen_addpatchref(cd, cd->mcodeptr,
2237                                                                         PATCHER_get_putstatic,
2238                                                                         (unresolved_field *) iptr->target);
2239
2240                                 if (showdisassemble) {
2241                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2242                                 }
2243
2244                                 a = 0;
2245
2246                         } else {
2247                                 fieldinfo *fi = iptr->val.a;
2248
2249                                 if (!fi->class->initialized) {
2250                                         codegen_addpatchref(cd, cd->mcodeptr,
2251                                                                                 PATCHER_clinit, fi->class);
2252
2253                                         if (showdisassemble) {
2254                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2255                                         }
2256                                 }
2257
2258                                 a = (ptrint) &(fi->value);
2259                         }
2260
2261                         /* This approach is much faster than moving the field address     */
2262                         /* inline into a register. */
2263                         a = dseg_addaddress(cd, a);
2264                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP2);
2265                         switch (iptr->op1) {
2266                         case TYPE_INT:
2267                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2268                                 x86_64_movl_membase_reg(cd, REG_ITMP2, 0, d);
2269                                 store_reg_to_var_int(iptr->dst, d);
2270                                 break;
2271                         case TYPE_LNG:
2272                         case TYPE_ADR:
2273                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2274                                 x86_64_mov_membase_reg(cd, REG_ITMP2, 0, d);
2275                                 store_reg_to_var_int(iptr->dst, d);
2276                                 break;
2277                         case TYPE_FLT:
2278                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2279                                 x86_64_movss_membase_reg(cd, REG_ITMP2, 0, d);
2280                                 store_reg_to_var_flt(iptr->dst, d);
2281                                 break;
2282                         case TYPE_DBL:
2283                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2284                                 x86_64_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2285                                 store_reg_to_var_flt(iptr->dst, d);
2286                                 break;
2287                         }
2288                         break;
2289
2290                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2291                                       /* op1 = type, val.a = field address            */
2292
2293                         if (!iptr->val.a) {
2294                                 codegen_addpatchref(cd, cd->mcodeptr,
2295                                                                         PATCHER_get_putstatic,
2296                                                                         (unresolved_field *) iptr->target);
2297
2298                                 if (showdisassemble) {
2299                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2300                                 }
2301
2302                                 a = 0;
2303
2304                         } else {
2305                                 fieldinfo *fi = iptr->val.a;
2306
2307                                 if (!fi->class->initialized) {
2308                                         codegen_addpatchref(cd, cd->mcodeptr,
2309                                                                                 PATCHER_clinit, fi->class);
2310
2311                                         if (showdisassemble) {
2312                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2313                                         }
2314                                 }
2315
2316                                 a = (ptrint) &(fi->value);
2317                         }
2318
2319                         /* This approach is much faster than moving the field address     */
2320                         /* inline into a register. */
2321                         a = dseg_addaddress(cd, a);
2322                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP2);
2323                         switch (iptr->op1) {
2324                         case TYPE_INT:
2325                                 var_to_reg_int(s2, src, REG_ITMP1);
2326                                 x86_64_movl_reg_membase(cd, s2, REG_ITMP2, 0);
2327                                 break;
2328                         case TYPE_LNG:
2329                         case TYPE_ADR:
2330                                 var_to_reg_int(s2, src, REG_ITMP1);
2331                                 x86_64_mov_reg_membase(cd, s2, REG_ITMP2, 0);
2332                                 break;
2333                         case TYPE_FLT:
2334                                 var_to_reg_flt(s2, src, REG_FTMP1);
2335                                 x86_64_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2336                                 break;
2337                         case TYPE_DBL:
2338                                 var_to_reg_flt(s2, src, REG_FTMP1);
2339                                 x86_64_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2340                                 break;
2341                         }
2342                         break;
2343
2344                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2345                                           /* val = value (in current instruction)     */
2346                                           /* op1 = type, val.a = field address (in    */
2347                                           /* following NOP)                           */
2348
2349                         if (!iptr[1].val.a) {
2350                                 codegen_addpatchref(cd, cd->mcodeptr,
2351                                                                         PATCHER_get_putstatic,
2352                                                                         (unresolved_field *) iptr[1].target);
2353
2354                                 if (showdisassemble) {
2355                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2356                                 }
2357
2358                                 a = 0;
2359
2360                         } else {
2361                                 fieldinfo *fi = iptr[1].val.a;
2362
2363                                 if (!fi->class->initialized) {
2364                                         codegen_addpatchref(cd, cd->mcodeptr,
2365                                                                                 PATCHER_clinit, fi->class);
2366
2367                                         if (showdisassemble) {
2368                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2369                                         }
2370                                 }
2371
2372                                 a = (ptrint) &(fi->value);
2373                         }
2374
2375                         /* This approach is much faster than moving the field address     */
2376                         /* inline into a register. */
2377                         a = dseg_addaddress(cd, a);
2378                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP1);
2379                         switch (iptr->op1) {
2380                         case TYPE_INT:
2381                         case TYPE_FLT:
2382                                 x86_64_movl_imm_membase(cd, iptr->val.i, REG_ITMP1, 0);
2383                                 break;
2384                         case TYPE_LNG:
2385                         case TYPE_ADR:
2386                         case TYPE_DBL:
2387                                 if (IS_IMM32(iptr->val.l)) {
2388                                         x86_64_mov_imm_membase(cd, iptr->val.l, REG_ITMP1, 0);
2389                                 } else {
2390                                         x86_64_movl_imm_membase(cd, iptr->val.l, REG_ITMP1, 0);
2391                                         x86_64_movl_imm_membase(cd, iptr->val.l >> 32, REG_ITMP1, 4);
2392                                 }
2393                                 break;
2394                         }
2395                         break;
2396
2397                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2398                                       /* op1 = type, val.i = field offset             */
2399
2400                         var_to_reg_int(s1, src, REG_ITMP1);
2401                         gen_nullptr_check(s1);
2402
2403                         if (!iptr->val.a) {
2404                                 codegen_addpatchref(cd, cd->mcodeptr,
2405                                                                         PATCHER_get_putfield,
2406                                                                         (unresolved_field *) iptr->target);
2407
2408                                 if (showdisassemble) {
2409                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2410                                 }
2411
2412                                 a = 0;
2413
2414                         } else {
2415                                 a = ((fieldinfo *) (iptr->val.a))->offset;
2416                         }
2417
2418                         switch (iptr->op1) {
2419                         case TYPE_INT:
2420                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2421                                 x86_64_movl_membase32_reg(cd, s1, a, d);
2422                                 store_reg_to_var_int(iptr->dst, d);
2423                                 break;
2424                         case TYPE_LNG:
2425                         case TYPE_ADR:
2426                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2427                                 x86_64_mov_membase32_reg(cd, s1, a, d);
2428                                 store_reg_to_var_int(iptr->dst, d);
2429                                 break;
2430                         case TYPE_FLT:
2431                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2432                                 x86_64_movss_membase32_reg(cd, s1, a, d);
2433                                 store_reg_to_var_flt(iptr->dst, d);
2434                                 break;
2435                         case TYPE_DBL:
2436                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2437                                 x86_64_movsd_membase32_reg(cd, s1, a, d);
2438                                 store_reg_to_var_flt(iptr->dst, d);
2439                                 break;
2440                         }
2441                         break;
2442
2443                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2444                                       /* op1 = type, val.i = field offset             */
2445
2446                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2447                         gen_nullptr_check(s1);
2448                         if (IS_INT_LNG_TYPE(iptr->op1)) {
2449                                 var_to_reg_int(s2, src, REG_ITMP2);
2450                         } else {
2451                                 var_to_reg_flt(s2, src, REG_FTMP2);
2452                         }
2453
2454                         if (!iptr->val.a) {
2455                                 codegen_addpatchref(cd, cd->mcodeptr,
2456                                                                         PATCHER_get_putfield,
2457                                                                         (unresolved_field *) iptr->target);
2458
2459                                 if (showdisassemble) {
2460                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2461                                 }
2462
2463                                 a = 0;
2464
2465                         } else {
2466                                 a = ((fieldinfo *) (iptr->val.a))->offset;
2467                         }
2468
2469                         switch (iptr->op1) {
2470                         case TYPE_INT:
2471                                 x86_64_movl_reg_membase32(cd, s2, s1, a);
2472                                 break;
2473                         case TYPE_LNG:
2474                         case TYPE_ADR:
2475                                 x86_64_mov_reg_membase32(cd, s2, s1, a);
2476                                 break;
2477                         case TYPE_FLT:
2478                                 x86_64_movss_reg_membase32(cd, s2, s1, a);
2479                                 break;
2480                         case TYPE_DBL:
2481                                 x86_64_movsd_reg_membase32(cd, s2, s1, a);
2482                                 break;
2483                         }
2484                         break;
2485
2486                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2487                                           /* val = value (in current instruction)     */
2488                                           /* op1 = type, val.a = field address (in    */
2489                                           /* following NOP)                           */
2490
2491                         var_to_reg_int(s1, src, REG_ITMP1);
2492                         gen_nullptr_check(s1);
2493
2494                         if (!iptr[1].val.a) {
2495                                 codegen_addpatchref(cd, cd->mcodeptr,
2496                                                                         PATCHER_putfieldconst,
2497                                                                         (unresolved_field *) iptr[1].target);
2498
2499                                 if (showdisassemble) {
2500                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2501                                 }
2502
2503                                 a = 0;
2504
2505                         } else {
2506                                 a = ((fieldinfo *) (iptr[1].val.a))->offset;
2507                         }
2508
2509                         switch (iptr->op1) {
2510                         case TYPE_INT:
2511                         case TYPE_FLT:
2512                                 x86_64_movl_imm_membase32(cd, iptr->val.i, s1, a);
2513                                 break;
2514                         case TYPE_LNG:
2515                         case TYPE_ADR:
2516                         case TYPE_DBL:
2517                                 /* We can only optimize the move, if the class is resolved.   */
2518                                 /* Otherwise we don't know what to patch.                     */
2519                                 if (iptr[1].val.a && IS_IMM32(iptr->val.l)) {
2520                                         x86_64_mov_imm_membase32(cd, iptr->val.l, s1, a);
2521                                 } else {
2522                                         x86_64_movl_imm_membase32(cd, iptr->val.l, s1, a);
2523                                         x86_64_movl_imm_membase32(cd, iptr->val.l >> 32, s1, a + 4);
2524                                 }
2525                                 break;
2526                         }
2527                         break;
2528
2529
2530                 /* branch operations **************************************************/
2531
2532                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2533
2534                         var_to_reg_int(s1, src, REG_ITMP1);
2535                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2536
2537                         x86_64_call_imm(cd, 0); /* passing exception pointer              */
2538                         x86_64_pop_reg(cd, REG_ITMP2_XPC);
2539
2540                         x86_64_mov_imm_reg(cd, (ptrint) asm_handle_exception, REG_ITMP3);
2541                         x86_64_jmp_reg(cd, REG_ITMP3);
2542                         break;
2543
2544                 case ICMD_GOTO:         /* ... ==> ...                                */
2545                                         /* op1 = target JavaVM pc                     */
2546
2547                         x86_64_jmp_imm(cd, 0);
2548                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2549                         break;
2550
2551                 case ICMD_JSR:          /* ... ==> ...                                */
2552                                         /* op1 = target JavaVM pc                     */
2553
2554                         x86_64_call_imm(cd, 0);
2555                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2556                         break;
2557
2558                 case ICMD_RET:          /* ... ==> ...                                */
2559                                         /* op1 = local variable                       */
2560
2561                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2562                         var_to_reg_int(s1, var, REG_ITMP1);
2563                         x86_64_jmp_reg(cd, s1);
2564                         break;
2565
2566                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2567                                         /* op1 = target JavaVM pc                     */
2568
2569                         if (src->flags & INMEMORY) {
2570                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2571
2572                         } else {
2573                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2574                         }
2575                         x86_64_jcc(cd, X86_64_CC_E, 0);
2576                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2577                         break;
2578
2579                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2580                                         /* op1 = target JavaVM pc                     */
2581
2582                         if (src->flags & INMEMORY) {
2583                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2584
2585                         } else {
2586                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2587                         }
2588                         x86_64_jcc(cd, X86_64_CC_NE, 0);
2589                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2590                         break;
2591
2592                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2593                                         /* op1 = target JavaVM pc, val.i = constant   */
2594
2595                         x86_64_emit_ifcc(cd, X86_64_CC_E, src, iptr);
2596                         break;
2597
2598                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2599                                         /* op1 = target JavaVM pc, val.i = constant   */
2600
2601                         x86_64_emit_ifcc(cd, X86_64_CC_L, src, iptr);
2602                         break;
2603
2604                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2605                                         /* op1 = target JavaVM pc, val.i = constant   */
2606
2607                         x86_64_emit_ifcc(cd, X86_64_CC_LE, src, iptr);
2608                         break;
2609
2610                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2611                                         /* op1 = target JavaVM pc, val.i = constant   */
2612
2613                         x86_64_emit_ifcc(cd, X86_64_CC_NE, src, iptr);
2614                         break;
2615
2616                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2617                                         /* op1 = target JavaVM pc, val.i = constant   */
2618
2619                         x86_64_emit_ifcc(cd, X86_64_CC_G, src, iptr);
2620                         break;
2621
2622                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2623                                         /* op1 = target JavaVM pc, val.i = constant   */
2624
2625                         x86_64_emit_ifcc(cd, X86_64_CC_GE, src, iptr);
2626                         break;
2627
2628                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2629                                         /* op1 = target JavaVM pc, val.l = constant   */
2630
2631                         x86_64_emit_if_lcc(cd, X86_64_CC_E, src, iptr);
2632                         break;
2633
2634                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2635                                         /* op1 = target JavaVM pc, val.l = constant   */
2636
2637                         x86_64_emit_if_lcc(cd, X86_64_CC_L, src, iptr);
2638                         break;
2639
2640                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2641                                         /* op1 = target JavaVM pc, val.l = constant   */
2642
2643                         x86_64_emit_if_lcc(cd, X86_64_CC_LE, src, iptr);
2644                         break;
2645
2646                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2647                                         /* op1 = target JavaVM pc, val.l = constant   */
2648
2649                         x86_64_emit_if_lcc(cd, X86_64_CC_NE, src, iptr);
2650                         break;
2651
2652                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2653                                         /* op1 = target JavaVM pc, val.l = constant   */
2654
2655                         x86_64_emit_if_lcc(cd, X86_64_CC_G, src, iptr);
2656                         break;
2657
2658                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2659                                         /* op1 = target JavaVM pc, val.l = constant   */
2660
2661                         x86_64_emit_if_lcc(cd, X86_64_CC_GE, src, iptr);
2662                         break;
2663
2664                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2665                                         /* op1 = target JavaVM pc                     */
2666
2667                         x86_64_emit_if_icmpcc(cd, X86_64_CC_E, src, iptr);
2668                         break;
2669
2670                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2671                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2672
2673                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_E, src, iptr);
2674                         break;
2675
2676                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2677                                         /* op1 = target JavaVM pc                     */
2678
2679                         x86_64_emit_if_icmpcc(cd, X86_64_CC_NE, src, iptr);
2680                         break;
2681
2682                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2683                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2684
2685                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_NE, src, iptr);
2686                         break;
2687
2688                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2689                                         /* op1 = target JavaVM pc                     */
2690
2691                         x86_64_emit_if_icmpcc(cd, X86_64_CC_L, src, iptr);
2692                         break;
2693
2694                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2695                                     /* op1 = target JavaVM pc                     */
2696
2697                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_L, src, iptr);
2698                         break;
2699
2700                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2701                                         /* op1 = target JavaVM pc                     */
2702
2703                         x86_64_emit_if_icmpcc(cd, X86_64_CC_G, src, iptr);
2704                         break;
2705
2706                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2707                                 /* op1 = target JavaVM pc                     */
2708
2709                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_G, src, iptr);
2710                         break;
2711
2712                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2713                                         /* op1 = target JavaVM pc                     */
2714
2715                         x86_64_emit_if_icmpcc(cd, X86_64_CC_LE, src, iptr);
2716                         break;
2717
2718                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2719                                         /* op1 = target JavaVM pc                     */
2720
2721                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_LE, src, iptr);
2722                         break;
2723
2724                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2725                                         /* op1 = target JavaVM pc                     */
2726
2727                         x86_64_emit_if_icmpcc(cd, X86_64_CC_GE, src, iptr);
2728                         break;
2729
2730                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2731                                     /* op1 = target JavaVM pc                     */
2732
2733                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_GE, src, iptr);
2734                         break;
2735
2736                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2737
2738                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2739                         break;
2740
2741                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2742                                         /* val.i = constant                           */
2743
2744                         var_to_reg_int(s1, src, REG_ITMP1);
2745                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2746                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2747                                 if (s1 == d) {
2748                                         M_INTMOVE(s1, REG_ITMP1);
2749                                         s1 = REG_ITMP1;
2750                                 }
2751                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2752                         }
2753                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2754                         x86_64_testl_reg_reg(cd, s1, s1);
2755                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_E, REG_ITMP2, d);
2756                         store_reg_to_var_int(iptr->dst, d);
2757                         break;
2758
2759                 case ICMD_IFNE_ICONST:  /* ..., value ==> ..., constant               */
2760                                         /* val.i = constant                           */
2761
2762                         var_to_reg_int(s1, src, REG_ITMP1);
2763                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2764                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2765                                 if (s1 == d) {
2766                                         M_INTMOVE(s1, REG_ITMP1);
2767                                         s1 = REG_ITMP1;
2768                                 }
2769                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2770                         }
2771                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2772                         x86_64_testl_reg_reg(cd, s1, s1);
2773                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_NE, REG_ITMP2, d);
2774                         store_reg_to_var_int(iptr->dst, d);
2775                         break;
2776
2777                 case ICMD_IFLT_ICONST:  /* ..., value ==> ..., constant               */
2778                                         /* val.i = constant                           */
2779
2780                         var_to_reg_int(s1, src, REG_ITMP1);
2781                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2782                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2783                                 if (s1 == d) {
2784                                         M_INTMOVE(s1, REG_ITMP1);
2785                                         s1 = REG_ITMP1;
2786                                 }
2787                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2788                         }
2789                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2790                         x86_64_testl_reg_reg(cd, s1, s1);
2791                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_L, REG_ITMP2, d);
2792                         store_reg_to_var_int(iptr->dst, d);
2793                         break;
2794
2795                 case ICMD_IFGE_ICONST:  /* ..., value ==> ..., constant               */
2796                                         /* val.i = constant                           */
2797
2798                         var_to_reg_int(s1, src, REG_ITMP1);
2799                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2800                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2801                                 if (s1 == d) {
2802                                         M_INTMOVE(s1, REG_ITMP1);
2803                                         s1 = REG_ITMP1;
2804                                 }
2805                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2806                         }
2807                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2808                         x86_64_testl_reg_reg(cd, s1, s1);
2809                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_GE, REG_ITMP2, d);
2810                         store_reg_to_var_int(iptr->dst, d);
2811                         break;
2812
2813                 case ICMD_IFGT_ICONST:  /* ..., value ==> ..., constant               */
2814                                         /* val.i = constant                           */
2815
2816                         var_to_reg_int(s1, src, REG_ITMP1);
2817                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2818                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2819                                 if (s1 == d) {
2820                                         M_INTMOVE(s1, REG_ITMP1);
2821                                         s1 = REG_ITMP1;
2822                                 }
2823                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2824                         }
2825                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2826                         x86_64_testl_reg_reg(cd, s1, s1);
2827                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP2, d);
2828                         store_reg_to_var_int(iptr->dst, d);
2829                         break;
2830
2831                 case ICMD_IFLE_ICONST:  /* ..., value ==> ..., constant               */
2832                                         /* val.i = constant                           */
2833
2834                         var_to_reg_int(s1, src, REG_ITMP1);
2835                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2836                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2837                                 if (s1 == d) {
2838                                         M_INTMOVE(s1, REG_ITMP1);
2839                                         s1 = REG_ITMP1;
2840                                 }
2841                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2842                         }
2843                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2844                         x86_64_testl_reg_reg(cd, s1, s1);
2845                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, d);
2846                         store_reg_to_var_int(iptr->dst, d);
2847                         break;
2848
2849
2850                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2851                 case ICMD_LRETURN:
2852                 case ICMD_ARETURN:
2853
2854                         var_to_reg_int(s1, src, REG_RESULT);
2855                         M_INTMOVE(s1, REG_RESULT);
2856
2857                         goto nowperformreturn;
2858
2859                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2860                 case ICMD_DRETURN:
2861
2862                         var_to_reg_flt(s1, src, REG_FRESULT);
2863                         M_FLTMOVE(s1, REG_FRESULT);
2864
2865                         goto nowperformreturn;
2866
2867                 case ICMD_RETURN:      /* ...  ==> ...                                */
2868
2869 nowperformreturn:
2870                         {
2871                         s4 i, p;
2872
2873                         p = parentargs_base;
2874
2875                         /* call trace function */
2876                         if (runverbose) {
2877                                 x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
2878
2879                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2880                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2881
2882                                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
2883                                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2884                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2885                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2886
2887                                 x86_64_mov_imm_reg(cd, (u8) builtin_displaymethodstop, REG_ITMP1);
2888                                 x86_64_call_reg(cd, REG_ITMP1);
2889
2890                                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2891                                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2892
2893                                 x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
2894                         }
2895
2896 #if defined(USE_THREADS)
2897                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2898                                 x86_64_mov_membase_reg(cd, REG_SP, rd->memuse * 8, rd->argintregs[0]);
2899
2900                                 /* we need to save the proper return value */
2901                                 switch (iptr->opc) {
2902                                 case ICMD_IRETURN:
2903                                 case ICMD_ARETURN:
2904                                 case ICMD_LRETURN:
2905                                         x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, rd->memuse * 8);
2906                                         break;
2907                                 case ICMD_FRETURN:
2908                                 case ICMD_DRETURN:
2909                                         x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, rd->memuse * 8);
2910                                         break;
2911                                 }
2912
2913                                 x86_64_mov_imm_reg(cd, (ptrint) builtin_monitorexit, REG_ITMP1);
2914                                 x86_64_call_reg(cd, REG_ITMP1);
2915
2916                                 /* and now restore the proper return value */
2917                                 switch (iptr->opc) {
2918                                 case ICMD_IRETURN:
2919                                 case ICMD_ARETURN:
2920                                 case ICMD_LRETURN:
2921                                         x86_64_mov_membase_reg(cd, REG_SP, rd->memuse * 8, REG_RESULT);
2922                                         break;
2923                                 case ICMD_FRETURN:
2924                                 case ICMD_DRETURN:
2925                                         x86_64_movq_membase_reg(cd, REG_SP, rd->memuse * 8, REG_FRESULT);
2926                                         break;
2927                                 }
2928                         }
2929 #endif
2930
2931                         /* restore saved registers                                        */
2932                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2933                                 p--; x86_64_mov_membase_reg(cd, REG_SP, p * 8, rd->savintregs[i]);
2934                         }
2935                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2936                                 p--; x86_64_movq_membase_reg(cd, REG_SP, p * 8, rd->savfltregs[i]);
2937                         }
2938
2939                         /* deallocate stack                                               */
2940                         if (parentargs_base) {
2941                                 x86_64_alu_imm_reg(cd, X86_64_ADD, parentargs_base * 8, REG_SP);
2942                         }
2943
2944                         x86_64_ret(cd);
2945                         }
2946                         break;
2947
2948
2949                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2950                         {
2951                                 s4 i, l, *s4ptr;
2952                                 void **tptr;
2953
2954                                 tptr = (void **) iptr->target;
2955
2956                                 s4ptr = iptr->val.a;
2957                                 l = s4ptr[1];                          /* low     */
2958                                 i = s4ptr[2];                          /* high    */
2959
2960                                 var_to_reg_int(s1, src, REG_ITMP1);
2961                                 M_INTMOVE(s1, REG_ITMP1);
2962                                 if (l != 0) {
2963                                         x86_64_alul_imm_reg(cd, X86_64_SUB, l, REG_ITMP1);
2964                                 }
2965                                 i = i - l + 1;
2966
2967                 /* range check */
2968                                 x86_64_alul_imm_reg(cd, X86_64_CMP, i - 1, REG_ITMP1);
2969                                 x86_64_jcc(cd, X86_64_CC_A, 0);
2970
2971                 /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[0]), cd->mcodeptr); */
2972                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
2973
2974                                 /* build jump table top down and use address of lowest entry */
2975
2976                 /* s4ptr += 3 + i; */
2977                                 tptr += i;
2978
2979                                 while (--i >= 0) {
2980                                         /* dseg_addtarget(cd, BlockPtrOfPC(*--s4ptr)); */
2981                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
2982                                         --tptr;
2983                                 }
2984
2985                                 /* length of dataseg after last dseg_addtarget is used by load */
2986
2987                                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2);
2988                                 dseg_adddata(cd, cd->mcodeptr);
2989                                 x86_64_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2990                                 x86_64_jmp_reg(cd, REG_ITMP1);
2991                         }
2992                         break;
2993
2994
2995                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2996                         {
2997                                 s4 i, l, val, *s4ptr;
2998                                 void **tptr;
2999
3000                                 tptr = (void **) iptr->target;
3001
3002                                 s4ptr = iptr->val.a;
3003                                 l = s4ptr[0];                          /* default  */
3004                                 i = s4ptr[1];                          /* count    */
3005
3006                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
3007                                 var_to_reg_int(s1, src, REG_ITMP1);    /* reg compare should always be faster */
3008                                 while (--i >= 0) {
3009                                         s4ptr += 2;
3010                                         ++tptr;
3011
3012                                         val = s4ptr[0];
3013                                         x86_64_alul_imm_reg(cd, X86_64_CMP, val, s1);
3014                                         x86_64_jcc(cd, X86_64_CC_E, 0);
3015                                         codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3016                                 }
3017
3018                                 x86_64_jmp_imm(cd, 0);
3019
3020                                 tptr = (void **) iptr->target;
3021                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3022                         }
3023                         break;
3024
3025
3026                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
3027                                         /* op1 = arg count val.a = builtintable entry */
3028
3029                         bte = iptr->val.a;
3030                         md = bte->md;
3031                         goto gen_method;
3032
3033                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
3034                                         /* op1 = arg count, val.a = method pointer    */
3035
3036                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
3037                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
3038                 case ICMD_INVOKEINTERFACE:
3039
3040                         lm = iptr->val.a;
3041
3042                         if (lm)
3043                                 md = lm->parseddesc;
3044                         else {
3045                                 unresolved_method *um = iptr->target;
3046                                 md = um->methodref->parseddesc.md;
3047                         }
3048
3049 gen_method:
3050                         s3 = iptr->op1;
3051
3052                         MCODECHECK((20 * s3) + 128);
3053
3054                         /* copy arguments to registers or stack location */
3055
3056                         for (s3 = s3 - 1; s3 >= 0; s3--, src = src->prev) {
3057                                 if (src->varkind == ARGVAR)
3058                                         continue;
3059                                 if (IS_INT_LNG_TYPE(src->type)) {
3060                                         if (!md->params[s3].inmemory) {
3061                                                 s1 = rd->argintregs[md->params[s3].regoff];
3062                                                 var_to_reg_int(d, src, s1);
3063                                                 M_INTMOVE(d, s1);
3064                                         } else {
3065                                                 var_to_reg_int(d, src, REG_ITMP1);
3066                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
3067                                         }
3068
3069                                 } else {
3070                                         if (!md->params[s3].inmemory) {
3071                                                 s1 = rd->argfltregs[md->params[s3].regoff];
3072                                                 var_to_reg_flt(d, src, s1);
3073                                                 M_FLTMOVE(d, s1);
3074                                         } else {
3075                                                 var_to_reg_flt(d, src, REG_FTMP1);
3076                                                 M_DST(d, REG_SP, md->params[s3].regoff * 8);
3077                                         }
3078                                 }
3079                         }
3080
3081                         switch (iptr->opc) {
3082                         case ICMD_BUILTIN:
3083                                 if (iptr->target) {
3084                                         codegen_addpatchref(cd, cd->mcodeptr,
3085                                                                                 bte->fp, iptr->target);
3086
3087                                         if (showdisassemble) {
3088                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3089                                         }
3090
3091                                         a = 0;
3092
3093                                 } else {
3094                                         a = (ptrint) bte->fp;
3095                                 }
3096
3097                                 d = md->returntype.type;
3098
3099                                 x86_64_mov_imm_reg(cd, a, REG_ITMP1);
3100                                 x86_64_call_reg(cd, REG_ITMP1);
3101                                 break;
3102
3103                         case ICMD_INVOKESPECIAL:
3104                                 x86_64_test_reg_reg(cd, rd->argintregs[0], rd->argintregs[0]);
3105                                 x86_64_jcc(cd, X86_64_CC_Z, 0);
3106                                 codegen_addxnullrefs(cd, cd->mcodeptr);
3107
3108                                 /* first argument contains pointer */
3109 /*                              gen_nullptr_check(rd->argintregs[0]); */
3110
3111                                 /* access memory for hardware nullptr */
3112 /*                              x86_64_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); */
3113
3114                                 /* fall through */
3115
3116                         case ICMD_INVOKESTATIC:
3117                                 if (!lm) {
3118                                         unresolved_method *um = iptr->target;
3119
3120                                         codegen_addpatchref(cd, cd->mcodeptr,
3121                                                                                 PATCHER_invokestatic_special, um);
3122
3123                                         if (showdisassemble) {
3124                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3125                                         }
3126
3127                                         a = 0;
3128                                         d = um->methodref->parseddesc.md->returntype.type;
3129
3130                                 } else {
3131                                         a = (ptrint) lm->stubroutine;
3132                                         d = lm->parseddesc->returntype.type;
3133                                 }
3134
3135                                 x86_64_mov_imm_reg(cd, a, REG_ITMP2);
3136                                 x86_64_call_reg(cd, REG_ITMP2);
3137                                 break;
3138
3139                         case ICMD_INVOKEVIRTUAL:
3140                                 gen_nullptr_check(rd->argintregs[0]);
3141
3142                                 if (!lm) {
3143                                         unresolved_method *um = iptr->target;
3144
3145                                         codegen_addpatchref(cd, cd->mcodeptr,
3146                                                                                 PATCHER_invokevirtual, um);
3147
3148                                         if (showdisassemble) {
3149                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3150                                         }
3151
3152                                         s1 = 0;
3153                                         d = um->methodref->parseddesc.md->returntype.type;
3154
3155                                 } else {
3156                                         s1 = OFFSET(vftbl_t, table[0]) +
3157                                                 sizeof(methodptr) * lm->vftblindex;
3158                                         d = lm->parseddesc->returntype.type;
3159                                 }
3160
3161                                 x86_64_mov_membase_reg(cd, rd->argintregs[0],
3162                                                                            OFFSET(java_objectheader, vftbl),
3163                                                                            REG_ITMP2);
3164                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP1);
3165                                 x86_64_call_reg(cd, REG_ITMP1);
3166                                 break;
3167
3168                         case ICMD_INVOKEINTERFACE:
3169                                 gen_nullptr_check(rd->argintregs[0]);
3170
3171                                 if (!lm) {
3172                                         unresolved_method *um = iptr->target;
3173
3174                                         codegen_addpatchref(cd, cd->mcodeptr,
3175                                                                                 PATCHER_invokeinterface, um);
3176
3177                                         if (showdisassemble) {
3178                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3179                                         }
3180
3181                                         s1 = 0;
3182                                         s2 = 0;
3183                                         d = um->methodref->parseddesc.md->returntype.type;
3184
3185                                 } else {
3186                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3187                                                 sizeof(methodptr) * lm->class->index;
3188
3189                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3190
3191                                         d = lm->parseddesc->returntype.type;
3192                                 }
3193
3194                                 x86_64_mov_membase_reg(cd, rd->argintregs[0],
3195                                                                            OFFSET(java_objectheader, vftbl),
3196                                                                            REG_ITMP2);
3197                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP2);
3198                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s2, REG_ITMP1);
3199                                 x86_64_call_reg(cd, REG_ITMP1);
3200                                 break;
3201                         }
3202
3203                         /* d contains return type */
3204
3205                         if (d != TYPE_VOID) {
3206                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3207                                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3208                                         M_INTMOVE(REG_RESULT, s1);
3209                                         store_reg_to_var_int(iptr->dst, s1);
3210                                 } else {
3211                                         s1 = reg_of_var(rd, iptr->dst, REG_FRESULT);
3212                                         M_FLTMOVE(REG_FRESULT, s1);
3213                                         store_reg_to_var_flt(iptr->dst, s1);
3214                                 }
3215                         }
3216                         break;
3217
3218
3219                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3220
3221                                       /* op1:   0 == array, 1 == class                */
3222                                       /* val.a: (classinfo *) superclass              */
3223
3224                         /*  superclass is an interface:
3225                          *
3226                          *  OK if ((sub == NULL) ||
3227                          *         (sub->vftbl->interfacetablelength > super->index) &&
3228                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3229                          *
3230                          *  superclass is a class:
3231                          *
3232                          *  OK if ((sub == NULL) || (0
3233                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3234                          *         super->vftbl->diffval));
3235                          */
3236
3237                         {
3238                         classinfo *super;
3239                         vftbl_t   *supervftbl;
3240                         s4         superindex;
3241
3242                         super = (classinfo *) iptr->val.a;
3243
3244                         if (!super) {
3245                                 superindex = 0;
3246                                 supervftbl = NULL;
3247
3248                         } else {
3249                                 superindex = super->index;
3250                                 supervftbl = super->vftbl;
3251                         }
3252
3253 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3254             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3255 #endif
3256                         var_to_reg_int(s1, src, REG_ITMP1);
3257
3258                         /* calculate interface checkcast code size */
3259
3260                         s2 = 3; /* mov_membase_reg */
3261                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3262
3263                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3264                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3265                                 3 /* test */ + 6 /* jcc */;
3266
3267                         if (!super)
3268                                 s2 += (showdisassemble ? 5 : 0);
3269
3270                         /* calculate class checkcast code size */
3271
3272                         s3 = 3; /* mov_membase_reg */
3273                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3274                         s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3275
3276 #if 0
3277                         if (s1 != REG_ITMP1) {
3278                                 a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3279                                 CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3280                                 a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3281                                 CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3282                                 a += 3;    /* sub */
3283
3284                         } else
3285 #endif
3286                                 {
3287                                         s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3288                                                 10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3289                                         CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3290                                 }
3291
3292                         s3 += 3 /* cmp */ + 6 /* jcc */;
3293
3294                         if (!super)
3295                                 s3 += (showdisassemble ? 5 : 0);
3296
3297                         /* if class is not resolved, check which code to call */
3298
3299                         if (!super) {
3300                                 x86_64_test_reg_reg(cd, s1, s1);
3301                                 x86_64_jcc(cd, X86_64_CC_Z, 6 + (showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3302
3303                                 codegen_addpatchref(cd, cd->mcodeptr,
3304                                                                         PATCHER_checkcast_instanceof_flags,
3305                                                                         (constant_classref *) iptr->target);
3306
3307                                 if (showdisassemble) {
3308                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3309                                 }
3310
3311                                 x86_64_movl_imm_reg(cd, 0, REG_ITMP2); /* super->flags */
3312                                 x86_64_alul_imm_reg(cd, X86_64_AND, ACC_INTERFACE, REG_ITMP2);
3313                                 x86_64_jcc(cd, X86_64_CC_Z, s2 + 5);
3314                         }
3315
3316                         /* interface checkcast code */
3317
3318                         if (!super || (super->flags & ACC_INTERFACE)) {
3319                                 if (super) {
3320                                         x86_64_test_reg_reg(cd, s1, s1);
3321                                         x86_64_jcc(cd, X86_64_CC_Z, s2);
3322                                 }
3323
3324                                 x86_64_mov_membase_reg(cd, s1,
3325                                                                            OFFSET(java_objectheader, vftbl),
3326                                                                            REG_ITMP2);
3327
3328                                 if (!super) {
3329                                         codegen_addpatchref(cd, cd->mcodeptr,
3330                                                                                 PATCHER_checkcast_instanceof_interface,
3331                                                                                 (constant_classref *) iptr->target);
3332
3333                                         if (showdisassemble) {
3334                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3335                                         }
3336                                 }
3337
3338                                 x86_64_movl_membase32_reg(cd, REG_ITMP2,
3339                                                                                   OFFSET(vftbl_t, interfacetablelength),
3340                                                                                   REG_ITMP3);
3341                                 x86_64_alu_imm32_reg(cd, X86_64_SUB, superindex, REG_ITMP3);
3342                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3343                                 x86_64_jcc(cd, X86_64_CC_LE, 0);
3344                                 codegen_addxcastrefs(cd, cd->mcodeptr);
3345                                 x86_64_mov_membase32_reg(cd, REG_ITMP2,
3346                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3347                                                                                  superindex * sizeof(methodptr*),
3348                                                                                  REG_ITMP3);
3349                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3350                                 x86_64_jcc(cd, X86_64_CC_E, 0);
3351                                 codegen_addxcastrefs(cd, cd->mcodeptr);
3352
3353                                 if (!super)
3354                                         x86_64_jmp_imm(cd, s3);
3355                         }
3356
3357                         /* class checkcast code */
3358
3359                         if (!super || !(super->flags & ACC_INTERFACE)) {
3360                                 if (super) {
3361                                         x86_64_test_reg_reg(cd, s1, s1);
3362                                         x86_64_jcc(cd, X86_64_CC_Z, s3);
3363                                 }
3364
3365                                 x86_64_mov_membase_reg(cd, s1,
3366                                                                            OFFSET(java_objectheader, vftbl),
3367                                                                            REG_ITMP2);
3368
3369                                 if (!super) {
3370                                         codegen_addpatchref(cd, cd->mcodeptr,
3371                                                                                 PATCHER_checkcast_class,
3372                                                                                 (constant_classref *) iptr->target);
3373
3374                                         if (showdisassemble) {
3375                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3376                                         }
3377                                 }
3378
3379                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP3);
3380 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3381                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3382 #endif
3383                                 x86_64_movl_membase32_reg(cd, REG_ITMP2,
3384                                                                                   OFFSET(vftbl_t, baseval),
3385                                                                                   REG_ITMP2);
3386 /*                                      if (s1 != REG_ITMP1) { */
3387 /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3388 /*                                                                                              OFFSET(vftbl_t, baseval), */
3389 /*                                                                                              REG_ITMP1); */
3390 /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3391 /*                                                                                              OFFSET(vftbl_t, diffval), */
3392 /*                                                                                              REG_ITMP3); */
3393 /*  #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
3394 /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3395 /*  #endif */
3396 /*                                              x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP1, REG_ITMP2); */
3397
3398 /*                                      } else { */
3399                                 x86_64_movl_membase32_reg(cd, REG_ITMP3,
3400                                                                                   OFFSET(vftbl_t, baseval),
3401                                                                                   REG_ITMP3);
3402                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP3, REG_ITMP2);
3403                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP3);
3404                                 x86_64_movl_membase_reg(cd, REG_ITMP3,
3405                                                                                 OFFSET(vftbl_t, diffval),
3406                                                                                 REG_ITMP3);
3407 /*                                      } */
3408 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3409                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3410 #endif
3411                                 x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP3, REG_ITMP2);
3412                                 x86_64_jcc(cd, X86_64_CC_A, 0);    /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3413                                 codegen_addxcastrefs(cd, cd->mcodeptr);
3414                         }
3415                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3416                         M_INTMOVE(s1, d);
3417                         store_reg_to_var_int(iptr->dst, d);
3418 /*                      if (iptr->dst->flags & INMEMORY) { */
3419 /*                              x86_64_mov_reg_membase(cd, s1, REG_SP, iptr->dst->regoff * 8); */
3420 /*                      } else { */
3421 /*                              M_INTMOVE(s1, iptr->dst->regoff); */
3422 /*                      } */
3423                         }
3424                         break;
3425
3426                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3427
3428                                       /* op1:   0 == array, 1 == class                */
3429                                       /* val.a: (classinfo *) superclass              */
3430
3431                         /*  superclass is an interface:
3432                          *
3433                          *  return (sub != NULL) &&
3434                          *         (sub->vftbl->interfacetablelength > super->index) &&
3435                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3436                          *
3437                          *  superclass is a class:
3438                          *
3439                          *  return ((sub != NULL) && (0
3440                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3441                          *          super->vftbl->diffvall));
3442                          */
3443
3444                         {
3445                         classinfo *super;
3446                         vftbl_t   *supervftbl;
3447                         s4         superindex;
3448
3449                         super = (classinfo *) iptr->val.a;
3450
3451                         if (!super) {
3452                                 superindex = 0;
3453                                 supervftbl = NULL;
3454
3455                         } else {
3456                                 superindex = super->index;
3457                                 supervftbl = super->vftbl;
3458                         }
3459
3460 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3461             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3462 #endif
3463
3464                         var_to_reg_int(s1, src, REG_ITMP1);
3465                         d = reg_of_var(rd, iptr->dst, REG_ITMP2);
3466                         if (s1 == d) {
3467                                 M_INTMOVE(s1, REG_ITMP1);
3468                                 s1 = REG_ITMP1;
3469                         }
3470
3471                         /* calculate interface instanceof code size */
3472
3473                         s2 = 3; /* mov_membase_reg */
3474                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3475                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3476                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3477                                 3 /* test */ + 4 /* setcc */;
3478
3479                         if (!super)
3480                                 s2 += (showdisassemble ? 5 : 0);
3481
3482                         /* calculate class instanceof code size */
3483
3484                         s3 = 3; /* mov_membase_reg */
3485                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3486                         s3 += 10; /* mov_imm_reg */
3487                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3488                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3489                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3490                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3491                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3492                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3493                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3494
3495                         if (!super)
3496                                 s3 += (showdisassemble ? 5 : 0);
3497
3498                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3499
3500                         /* if class is not resolved, check which code to call */
3501
3502                         if (!super) {
3503                                 x86_64_test_reg_reg(cd, s1, s1);
3504                                 x86_64_jcc(cd, X86_64_CC_Z, (6 + (showdisassemble ? 5 : 0) +
3505                                                                                          7 + 6 + s2 + 5 + s3));
3506
3507                                 codegen_addpatchref(cd, cd->mcodeptr,
3508                                                                         PATCHER_checkcast_instanceof_flags,
3509                                                                         (constant_classref *) iptr->target);
3510
3511                                 if (showdisassemble) {
3512                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3513                                 }
3514
3515                                 x86_64_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3516                                 x86_64_alul_imm_reg(cd, X86_64_AND, ACC_INTERFACE, REG_ITMP3);
3517                                 x86_64_jcc(cd, X86_64_CC_Z, s2 + 5);
3518                         }
3519
3520                         /* interface instanceof code */
3521
3522                         if (!super || (super->flags & ACC_INTERFACE)) {
3523                                 if (super) {
3524                                         x86_64_test_reg_reg(cd, s1, s1);
3525                                         x86_64_jcc(cd, X86_64_CC_Z, s2);
3526                                 }
3527
3528                                 x86_64_mov_membase_reg(cd, s1,
3529                                                                            OFFSET(java_objectheader, vftbl),
3530                                                                            REG_ITMP1);
3531                                 if (!super) {
3532                                         codegen_addpatchref(cd, cd->mcodeptr,
3533                                                                                 PATCHER_checkcast_instanceof_interface,
3534                                                                                 (constant_classref *) iptr->target);
3535
3536                                         if (showdisassemble) {
3537                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3538                                         }
3539                                 }
3540
3541                                 x86_64_movl_membase32_reg(cd, REG_ITMP1,
3542                                                                                   OFFSET(vftbl_t, interfacetablelength),
3543                                                                                   REG_ITMP3);
3544                                 x86_64_alu_imm32_reg(cd, X86_64_SUB, superindex, REG_ITMP3);
3545                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3546
3547                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3548
3549                                 x86_64_jcc(cd, X86_64_CC_LE, a);
3550                                 x86_64_mov_membase32_reg(cd, REG_ITMP1,
3551                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3552                                                                                  superindex * sizeof(methodptr*),
3553                                                                                  REG_ITMP1);
3554                                 x86_64_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3555                                 x86_64_setcc_reg(cd, X86_64_CC_NE, d);
3556
3557                                 if (!super)
3558                                         x86_64_jmp_imm(cd, s3);
3559                         }
3560
3561                         /* class instanceof code */
3562
3563                         if (!super || !(super->flags & ACC_INTERFACE)) {
3564                                 if (super) {
3565                                         x86_64_test_reg_reg(cd, s1, s1);
3566                                         x86_64_jcc(cd, X86_64_CC_E, s3);
3567                                 }
3568
3569                                 x86_64_mov_membase_reg(cd, s1,
3570                                                                            OFFSET(java_objectheader, vftbl),
3571                                                                            REG_ITMP1);
3572
3573                                 if (!super) {
3574                                         codegen_addpatchref(cd, cd->mcodeptr,
3575                                                                                 PATCHER_instanceof_class,
3576                                                                                 (constant_classref *) iptr->target);
3577
3578                                         if (showdisassemble) {
3579                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3580                                         }
3581                                 }
3582
3583                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3584 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3585                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3586 #endif
3587                                 x86_64_movl_membase_reg(cd, REG_ITMP1,
3588                                                                                 OFFSET(vftbl_t, baseval),
3589                                                                                 REG_ITMP1);
3590                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3591                                                                                 OFFSET(vftbl_t, diffval),
3592                                                                                 REG_ITMP3);
3593                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3594                                                                                 OFFSET(vftbl_t, baseval),
3595                                                                                 REG_ITMP2);
3596 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3597                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3598 #endif
3599                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
3600                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d); /* may be REG_ITMP2 */
3601                                 x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP3, REG_ITMP1);
3602                                 x86_64_setcc_reg(cd, X86_64_CC_BE, d);
3603                         }
3604                         store_reg_to_var_int(iptr->dst, d);
3605                         }
3606                         break;
3607
3608                 case ICMD_CHECKASIZE:  /* ..., size ==> ..., size                     */
3609
3610                         if (src->flags & INMEMORY) {
3611                                 x86_64_alul_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
3612
3613                         } else {
3614                                 x86_64_testl_reg_reg(cd, src->regoff, src->regoff);
3615                         }
3616                         x86_64_jcc(cd, X86_64_CC_L, 0);
3617                         codegen_addxcheckarefs(cd, cd->mcodeptr);
3618                         break;
3619
3620                 case ICMD_CHECKEXCEPTION:    /* ... ==> ...                           */
3621
3622                         x86_64_test_reg_reg(cd, REG_RESULT, REG_RESULT);
3623                         x86_64_jcc(cd, X86_64_CC_E, 0);
3624                         codegen_addxexceptionrefs(cd, cd->mcodeptr);
3625                         break;
3626
3627                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3628                                          /* op1 = dimension, val.a = array descriptor */
3629
3630                         /* check for negative sizes and copy sizes to stack if necessary  */
3631
3632                         MCODECHECK((10 * 4 * iptr->op1) + 5 + 10 * 8);
3633
3634                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3635                                 var_to_reg_int(s2, src, REG_ITMP1);
3636                                 x86_64_testl_reg_reg(cd, s2, s2);
3637                                 x86_64_jcc(cd, X86_64_CC_L, 0);
3638                                 codegen_addxcheckarefs(cd, cd->mcodeptr);
3639
3640                                 /* copy SAVEDVAR sizes to stack */
3641
3642                                 if (src->varkind != ARGVAR) {
3643                                         x86_64_mov_reg_membase(cd, s2, REG_SP, s1 * 8);
3644                                 }
3645                         }
3646
3647                         /* is a patcher function set? */
3648
3649                         if (iptr->target) {
3650                                 codegen_addpatchref(cd, cd->mcodeptr,
3651                                                                         (functionptr) (ptrint) iptr->target,
3652                                                                         iptr->val.a);
3653
3654                                 if (showdisassemble) {
3655                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3656                                 }
3657
3658                                 a = 0;
3659
3660                         } else {
3661                                 a = (ptrint) iptr->val.a;
3662                         }
3663
3664                         /* a0 = dimension count */
3665
3666                         x86_64_mov_imm_reg(cd, iptr->op1, rd->argintregs[0]);
3667
3668                         /* a1 = arrayvftbl */
3669
3670                         x86_64_mov_imm_reg(cd, (ptrint) iptr->val.a, rd->argintregs[1]);
3671
3672                         /* a2 = pointer to dimensions = stack pointer */
3673
3674                         x86_64_mov_reg_reg(cd, REG_SP, rd->argintregs[2]);
3675
3676                         x86_64_mov_imm_reg(cd, (ptrint) BUILTIN_multianewarray, REG_ITMP1);
3677                         x86_64_call_reg(cd, REG_ITMP1);
3678
3679                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3680                         M_INTMOVE(REG_RESULT, s1);
3681                         store_reg_to_var_int(iptr->dst, s1);
3682                         break;
3683
3684                 default:
3685                         throw_cacao_exception_exit(string_java_lang_InternalError,
3686                                                                            "Unknown ICMD %d", iptr->opc);
3687         } /* switch */
3688
3689         } /* for instruction */
3690
3691         /* copy values to interface registers */
3692
3693         src = bptr->outstack;
3694         len = bptr->outdepth;
3695         MCODECHECK(512);
3696 #ifdef LSRA
3697         if (!opt_lsra)
3698 #endif
3699         while (src) {
3700                 len--;
3701                 if ((src->varkind != STACKVAR)) {
3702                         s2 = src->type;
3703                         if (IS_FLT_DBL_TYPE(s2)) {
3704                                 var_to_reg_flt(s1, src, REG_FTMP1);
3705                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3706                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3707
3708                                 } else {
3709                                         x86_64_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3710                                 }
3711
3712                         } else {
3713                                 var_to_reg_int(s1, src, REG_ITMP1);
3714                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3715                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3716
3717                                 } else {
3718                                         x86_64_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3719                                 }
3720                         }
3721                 }
3722                 src = src->prev;
3723         }
3724         } /* if (bptr -> flags >= BBREACHED) */
3725         } /* for basic block */
3726
3727         codegen_createlinenumbertable(cd);
3728
3729         {
3730
3731         /* generate ArrayIndexOutOfBoundsException stubs */
3732
3733         u1 *xcodeptr = NULL;
3734         branchref *bref;
3735
3736         for (bref = cd->xboundrefs; bref != NULL; bref = bref->next) {
3737                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3738                                   bref->branchpos,
3739                                                   cd->mcodeptr - cd->mcodebase);
3740
3741                 MCODECHECK(512);
3742
3743                 /* move index register into REG_ITMP1 */
3744
3745                 M_MOV(bref->reg, REG_ITMP1);                             /* 3 bytes  */
3746
3747                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                 /* 10 bytes */
3748                 dseg_adddata(cd, cd->mcodeptr);
3749                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);   /* 10 bytes */
3750                 M_AADD(REG_ITMP3, REG_ITMP2_XPC);                         /* 3 bytes  */
3751
3752                 if (xcodeptr != NULL) {
3753                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3754
3755                 } else {
3756                         xcodeptr = cd->mcodeptr;
3757
3758
3759                         /*create stackinfo -- begin*/
3760                         x86_64_alu_imm_reg(cd, X86_64_SUB, 4 * 8, REG_SP);
3761                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 3 * 8);
3762                         x86_64_mov_imm_membase(cd,0,REG_SP,2*8);
3763                         x86_64_mov_imm_membase(cd,0,REG_SP,1*8);
3764                         x86_64_mov_imm_reg(cd,(ptrint)asm_prepare_native_stackinfo,REG_ITMP3);
3765                         x86_64_call_reg(cd,REG_ITMP3);
3766                         /*create stackinfo -- end*/
3767
3768                         x86_64_mov_reg_reg(cd, REG_ITMP1, rd->argintregs[0]);
3769                         x86_64_mov_imm_reg(cd, (ptrint) new_arrayindexoutofboundsexception, REG_ITMP3);
3770                         x86_64_call_reg(cd, REG_ITMP3);
3771
3772                         /*remove stackinfo -- begin*/
3773                         x86_64_mov_imm_reg(cd,(ptrint)asm_remove_native_stackinfo,REG_ITMP3);
3774                         x86_64_call_reg(cd,REG_ITMP3);
3775                         /*remove stackinfo -- end*/
3776
3777                         x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, REG_ITMP2_XPC);
3778                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3779
3780                         x86_64_mov_imm_reg(cd, (ptrint) asm_handle_exception, REG_ITMP3);
3781                         x86_64_jmp_reg(cd, REG_ITMP3);
3782                 }
3783         }
3784
3785         /* generate NegativeArraySizeException stubs */
3786
3787         xcodeptr = NULL;
3788
3789         for (bref = cd->xcheckarefs; bref != NULL; bref = bref->next) {
3790                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3791                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3792                                                           bref->branchpos,
3793                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3794                         continue;
3795                 }
3796
3797                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3798                                   bref->branchpos,
3799                                                   cd->mcodeptr - cd->mcodebase);
3800
3801                 MCODECHECK(512);
3802
3803                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                 /* 10 bytes */
3804                 dseg_adddata(cd, cd->mcodeptr);
3805                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);   /* 10 bytes */
3806                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC); /* 3 bytes  */
3807
3808                 if (xcodeptr != NULL) {
3809                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3810
3811                 } else {
3812                         xcodeptr = cd->mcodeptr;
3813
3814
3815                         /*create stackinfo -- begin*/
3816                         x86_64_alu_imm_reg(cd, X86_64_SUB, 4 * 8, REG_SP);
3817                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 3 * 8);
3818                         x86_64_mov_imm_membase(cd,0,REG_SP,2*8);
3819                         x86_64_mov_imm_membase(cd,0,REG_SP,1*8);
3820                         x86_64_mov_imm_reg(cd,(ptrint)asm_prepare_native_stackinfo,REG_ITMP3);
3821                         x86_64_call_reg(cd,REG_ITMP3);
3822                         /*create stackinfo -- end*/
3823
3824                         x86_64_mov_imm_reg(cd, (u8) new_negativearraysizeexception, REG_ITMP3);
3825                         x86_64_call_reg(cd, REG_ITMP3);
3826
3827                         /*remove stackinfo -- begin*/
3828                         x86_64_mov_imm_reg(cd,(ptrint)asm_remove_native_stackinfo,REG_ITMP3);
3829                         x86_64_call_reg(cd,REG_ITMP3);
3830                         /*remove stackinfo -- end*/
3831
3832                         x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, REG_ITMP2_XPC);
3833                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3834
3835                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
3836                         x86_64_jmp_reg(cd, REG_ITMP3);
3837                 }
3838         }
3839
3840         /* generate cast check stubs */
3841
3842         xcodeptr = NULL;
3843
3844         for (bref = cd->xcastrefs; bref != NULL; bref = bref->next) {
3845                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3846                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3847                                                           bref->branchpos,
3848                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3849                         continue;
3850                 }
3851
3852                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3853                                   bref->branchpos,
3854                                                   cd->mcodeptr - cd->mcodebase);
3855
3856                 MCODECHECK(512);
3857
3858                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                 /* 10 bytes */
3859                 dseg_adddata(cd, cd->mcodeptr);
3860                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);   /* 10 bytes */
3861                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC); /* 3 bytes  */
3862
3863                 if (xcodeptr != NULL) {
3864                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3865
3866                 } else {
3867                         xcodeptr = cd->mcodeptr;
3868
3869                         /*create stackinfo -- begin*/
3870                         x86_64_alu_imm_reg(cd, X86_64_SUB, 4 * 8, REG_SP);
3871                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 3 * 8);
3872                         x86_64_mov_imm_membase(cd,0,REG_SP,2*8);
3873                         x86_64_mov_imm_membase(cd,0,REG_SP,1*8);
3874                         x86_64_mov_imm_reg(cd,(ptrint)asm_prepare_native_stackinfo,REG_ITMP3);
3875                         x86_64_call_reg(cd,REG_ITMP3);
3876                         /*create stackinfo -- end*/
3877
3878
3879                         x86_64_mov_imm_reg(cd, (u8) new_classcastexception, REG_ITMP3);
3880                         x86_64_call_reg(cd, REG_ITMP3);
3881
3882                         /*remove stackinfo -- begin*/
3883                         x86_64_mov_imm_reg(cd,(ptrint)asm_remove_native_stackinfo,REG_ITMP3);
3884                         x86_64_call_reg(cd,REG_ITMP3);
3885                         /*remove stackinfo -- end*/
3886
3887                         x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, REG_ITMP2_XPC);
3888                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3889
3890                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
3891                         x86_64_jmp_reg(cd, REG_ITMP3);
3892                 }
3893         }
3894
3895         /* generate divide by zero check stubs */
3896
3897         xcodeptr = NULL;
3898
3899         for (bref = cd->xdivrefs; bref != NULL; bref = bref->next) {
3900                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3901                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3902                                                           bref->branchpos,
3903                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3904                         continue;
3905                 }
3906
3907                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3908                                   bref->branchpos,
3909                                                   cd->mcodeptr - cd->mcodebase);
3910
3911                 MCODECHECK(512);
3912
3913                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                 /* 10 bytes */
3914                 dseg_adddata(cd, cd->mcodeptr);
3915                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);   /* 10 bytes */
3916                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC); /* 3 bytes  */
3917
3918                 if (xcodeptr != NULL) {
3919                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3920
3921                 } else {
3922                         xcodeptr = cd->mcodeptr;
3923
3924                         /*create stackinfo -- begin*/
3925                         x86_64_alu_imm_reg(cd, X86_64_SUB, 4 * 8, REG_SP);
3926                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 3 * 8);
3927                         x86_64_mov_imm_membase(cd,0,REG_SP,2*8);
3928                         x86_64_mov_imm_membase(cd,0,REG_SP,1*8);
3929                         x86_64_mov_imm_reg(cd,(ptrint)asm_prepare_native_stackinfo,REG_ITMP3);
3930                         x86_64_call_reg(cd,REG_ITMP3);
3931                         /*create stackinfo -- end*/
3932
3933                         x86_64_mov_imm_reg(cd, (u8) new_arithmeticexception, REG_ITMP3);
3934                         x86_64_call_reg(cd, REG_ITMP3);
3935
3936                         /*remove stackinfo -- begin*/
3937                         x86_64_mov_imm_reg(cd,(ptrint)asm_remove_native_stackinfo,REG_ITMP3);
3938                         x86_64_call_reg(cd,REG_ITMP3);
3939                         /*remove stackinfo -- end*/
3940
3941                         x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, REG_ITMP2_XPC);
3942                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3943
3944                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
3945                         x86_64_jmp_reg(cd, REG_ITMP3);
3946                 }
3947         }
3948
3949         /* generate exception check stubs */
3950
3951         xcodeptr = NULL;
3952
3953         for (bref = cd->xexceptionrefs; bref != NULL; bref = bref->next) {
3954                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3955                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3956                                                           bref->branchpos,
3957                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3958                         continue;
3959                 }
3960
3961                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3962                                   bref->branchpos,
3963                                                   cd->mcodeptr - cd->mcodebase);
3964
3965                 MCODECHECK(512);
3966
3967                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                 /* 10 bytes */
3968                 dseg_adddata(cd, cd->mcodeptr);
3969                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP1);   /* 10 bytes */
3970                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP1, REG_ITMP2_XPC); /* 3 bytes  */
3971
3972                 if (xcodeptr != NULL) {
3973                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3974
3975                 } else {
3976                         xcodeptr = cd->mcodeptr;
3977
3978                         x86_64_alu_imm_reg(cd, X86_64_SUB, 4*8, REG_SP);
3979                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 3*8);
3980                         x86_64_mov_imm_membase(cd, 0, REG_SP, 2*8);
3981                         x86_64_mov_imm_membase(cd, 0, REG_SP, 1*8);
3982                         x86_64_mov_imm_membase(cd, 0, REG_SP, 0*8);
3983                         x86_64_mov_imm_reg(cd,(u8) asm_prepare_native_stackinfo,REG_ITMP1);
3984                         x86_64_call_reg(cd,REG_ITMP1);
3985
3986
3987 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3988                         x86_64_mov_imm_reg(cd, (u8) &builtin_get_exceptionptrptr, REG_ITMP1);
3989                         x86_64_call_reg(cd, REG_ITMP1);
3990                         x86_64_mov_membase_reg(cd, REG_RESULT, 0, REG_ITMP3);
3991                         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
3992                         x86_64_mov_reg_reg(cd, REG_ITMP3, REG_ITMP1_XPTR);
3993 #else
3994                         x86_64_mov_imm_reg(cd, (u8) &_exceptionptr, REG_ITMP3);
3995                         x86_64_mov_membase_reg(cd, REG_ITMP3, 0, REG_ITMP1_XPTR);
3996                         x86_64_mov_imm_membase(cd, 0, REG_ITMP3, 0);
3997 #endif
3998                         x86_64_mov_reg_reg(cd,REG_ITMP1_XPTR,RDI);
3999                         x86_64_mov_imm_reg(cd,(u8) helper_fillin_stacktrace_always,REG_ITMP1);
4000                         x86_64_call_reg(cd,REG_ITMP1);
4001                         x86_64_mov_reg_reg(cd,REG_RESULT,REG_ITMP1_XPTR);
4002
4003                         x86_64_mov_imm_reg(cd,(u8) asm_remove_native_stackinfo,REG_ITMP2);
4004                         x86_64_call_reg(cd,REG_ITMP2);
4005
4006                         x86_64_alu_imm_reg(cd, X86_64_ADD, 8, REG_SP);
4007                         x86_64_mov_membase_reg(cd, REG_SP, 0, REG_ITMP2_XPC);
4008                         x86_64_alu_imm_reg(cd, X86_64_ADD, 8, REG_SP);
4009
4010
4011                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
4012                         x86_64_jmp_reg(cd, REG_ITMP3);
4013                 }
4014         }
4015
4016         /* generate NullpointerException stubs */
4017
4018         xcodeptr = NULL;
4019
4020         for (bref = cd->xnullrefs; bref != NULL; bref = bref->next) {
4021                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
4022                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
4023                                                           bref->branchpos,
4024                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
4025                         continue;
4026                 }
4027
4028                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
4029                                   bref->branchpos,
4030                                                   cd->mcodeptr - cd->mcodebase);
4031
4032                 MCODECHECK(512);
4033
4034                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                 /* 10 bytes */
4035                 dseg_adddata(cd, cd->mcodeptr);
4036                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP1);   /* 10 bytes */
4037                 M_AADD(REG_ITMP1, REG_ITMP2_XPC);                         /* 3 bytes  */
4038
4039                 if (xcodeptr != NULL) {
4040                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
4041
4042                 } else {
4043                         xcodeptr = cd->mcodeptr;
4044
4045                         /*create stackinfo -- begin*/
4046                         x86_64_alu_imm_reg(cd, X86_64_SUB, 4 * 8, REG_SP);
4047                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 3 * 8);
4048                         x86_64_mov_imm_membase(cd,0,REG_SP,2*8);
4049                         x86_64_mov_imm_membase(cd,0,REG_SP,1*8);
4050                         x86_64_mov_imm_reg(cd,(ptrint)asm_prepare_native_stackinfo,REG_ITMP3);
4051                         x86_64_call_reg(cd,REG_ITMP3);
4052                         /*create stackinfo -- end*/
4053
4054
4055                         x86_64_mov_imm_reg(cd, (ptrint) new_nullpointerexception, REG_ITMP3);
4056                         x86_64_call_reg(cd, REG_ITMP3);
4057
4058                         /*remove stackinfo -- begin*/
4059                         x86_64_mov_imm_reg(cd,(ptrint)asm_remove_native_stackinfo,REG_ITMP3);
4060                         x86_64_call_reg(cd,REG_ITMP3);
4061                         /*remove stackinfo -- end*/
4062
4063                         x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, REG_ITMP2_XPC);
4064                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
4065
4066                         x86_64_mov_imm_reg(cd, (ptrint) asm_handle_exception, REG_ITMP3);
4067                         x86_64_jmp_reg(cd, REG_ITMP3);
4068                 }
4069         }
4070
4071         /* generate code patching stub call code */
4072
4073         {
4074                 patchref    *pref;
4075                 codegendata *tmpcd;
4076                 ptrint       mcode;
4077
4078                 tmpcd = DNEW(codegendata);
4079
4080                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4081                         /* check size of code segment */
4082
4083                         MCODECHECK(512);
4084
4085                         /* Get machine code which is patched back in later. A             */
4086                         /* `call rel32' is 5 bytes long (but read 8 bytes).               */
4087
4088                         xcodeptr = cd->mcodebase + pref->branchpos;
4089                         mcode = *((ptrint *) xcodeptr);
4090
4091                         /* patch in `call rel32' to call the following code               */
4092
4093                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
4094                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
4095
4096                         /* move pointer to java_objectheader onto stack */
4097
4098 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4099                         /* create a virtual java_objectheader */
4100
4101                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4102                         a = dseg_addaddress(cd, NULL);                      /* vftbl      */
4103
4104                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP3);
4105                         x86_64_push_reg(cd, REG_ITMP3);
4106 #else
4107                         x86_64_push_imm(cd, 0);
4108 #endif
4109
4110                         /* move machine code bytes and classinfo pointer into registers */
4111
4112                         x86_64_mov_imm_reg(cd, (ptrint) mcode, REG_ITMP3);
4113                         x86_64_push_reg(cd, REG_ITMP3);
4114                         x86_64_mov_imm_reg(cd, (ptrint) pref->ref, REG_ITMP3);
4115                         x86_64_push_reg(cd, REG_ITMP3);
4116
4117                         x86_64_mov_imm_reg(cd, (ptrint) pref->patcher, REG_ITMP3);
4118                         x86_64_push_reg(cd, REG_ITMP3);
4119
4120                         x86_64_mov_imm_reg(cd, (ptrint) asm_wrapper_patcher, REG_ITMP3);
4121                         x86_64_jmp_reg(cd, REG_ITMP3);
4122                 }
4123         }
4124         }
4125
4126         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
4127 }
4128
4129
4130 /* createcompilerstub **********************************************************
4131
4132    Creates a stub routine which calls the compiler.
4133
4134 *******************************************************************************/
4135
4136 #define COMPILERSTUB_SIZE    23
4137
4138 functionptr createcompilerstub(methodinfo *m)
4139 {
4140         u1          *s;                     /* memory to hold the stub            */
4141         codegendata *cd;
4142         s4           dumpsize;
4143
4144         s = CNEW(u1, COMPILERSTUB_SIZE);
4145
4146         /* mark start of dump memory area */
4147
4148         dumpsize = dump_size();
4149
4150         cd = DNEW(codegendata);
4151         cd->mcodeptr = s;
4152
4153         /* code for the stub */
4154
4155         x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP1); /* pass method to compiler */
4156         x86_64_mov_imm_reg(cd, (ptrint) asm_call_jit_compiler, REG_ITMP3);
4157         x86_64_jmp_reg(cd, REG_ITMP3);
4158
4159 #if defined(STATISTICS)
4160         if (opt_stat)
4161                 count_cstub_len += COMPILERSTUB_SIZE;
4162 #endif
4163
4164         /* release dump area */
4165
4166         dump_release(dumpsize);
4167
4168         return (functionptr) (ptrint) s;
4169 }
4170
4171
4172 /* createnativestub ************************************************************
4173
4174    Creates a stub routine which calls a native method.
4175
4176 *******************************************************************************/
4177
4178 functionptr createnativestub(functionptr f, methodinfo *m, codegendata *cd,
4179                                                          registerdata *rd, methoddesc *nmd)
4180 {
4181         methoddesc *md;
4182         s4          stackframesize;         /* size of stackframe if needed       */
4183         s4          nativeparams;
4184         s4          i, j;                   /* count variables                    */
4185         s4          t;
4186         s4          s1, s2, off;
4187
4188         /* initialize variables */
4189
4190         md = m->parseddesc;
4191         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
4192
4193
4194         /* calculate stack frame size */
4195
4196         stackframesize = 4 + INT_ARG_CNT + FLT_ARG_CNT + nmd->memuse;
4197
4198         if (!(stackframesize & 0x1))                /* keep stack 16-byte aligned */
4199                 stackframesize++;
4200
4201
4202         /* create method header */
4203
4204         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
4205         (void) dseg_adds4(cd, stackframesize * 8);              /* FrameSize      */
4206         (void) dseg_adds4(cd, 0);                               /* IsSync         */
4207         (void) dseg_adds4(cd, 0);                               /* IsLeaf         */
4208         (void) dseg_adds4(cd, 0);                               /* IntSave        */
4209         (void) dseg_adds4(cd, 0);                               /* FltSave        */
4210         (void) dseg_addlinenumbertablesize(cd);
4211         (void) dseg_adds4(cd, 0);                               /* ExTableSize    */
4212
4213
4214         /* initialize mcode variables */
4215
4216         cd->mcodeptr = (u1 *) cd->mcodebase;
4217         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
4218
4219
4220         /* generate stub code */
4221
4222         M_ASUB_IMM(stackframesize * 8, REG_SP);
4223
4224         /* if function is static, check for initialized */
4225
4226         if ((m->flags & ACC_STATIC) && !m->class->initialized) {
4227                 codegen_addpatchref(cd, cd->mcodeptr, PATCHER_clinit, m->class);
4228
4229                 if (showdisassemble) {
4230                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
4231                 }
4232         }
4233
4234         if (runverbose) {
4235                 /* save integer and float argument registers */
4236
4237                 for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4238                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4239                                 M_LST(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4240
4241                 for (i = 0, j = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4242                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4243                                 M_DST(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4244
4245                 /* show integer hex code for float arguments */
4246
4247                 for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++) {
4248                         /* if the paramtype is a float, we have to right shift all        */
4249                         /* following integer registers                                    */
4250
4251                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type)) {
4252                                 for (s1 = INT_ARG_CNT - 2; s1 >= i; s1--)
4253                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
4254
4255                                 x86_64_movd_freg_reg(cd, rd->argfltregs[j], rd->argintregs[i]);
4256                                 j++;
4257                         }
4258                 }
4259
4260                 x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP1);
4261                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, 0 * 8);
4262                 x86_64_mov_imm_reg(cd, (ptrint) builtin_trace_args, REG_ITMP1);
4263                 x86_64_call_reg(cd, REG_ITMP1);
4264
4265                 /* restore integer and float argument registers */
4266
4267                 for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4268                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4269                                 M_LLD(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4270
4271                 for (i = 0, j = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4272                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4273                                 M_DLD(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4274         }
4275
4276
4277         /* save integer and float argument registers */
4278
4279         for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4280                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4281                         M_LST(rd->argintregs[j++], REG_SP, i * 8);
4282
4283         for (i = 0, j = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4284                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4285                         M_DST(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4286
4287         /* create dynamic stack info */
4288
4289         x86_64_mov_imm_membase(cd, 0, REG_SP, (stackframesize - 1) * 8);
4290         x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP1);
4291         M_AST(REG_ITMP1, REG_SP, (stackframesize - 2) * 8);
4292         x86_64_mov_imm_reg(cd, (ptrint) builtin_asm_get_stackframeinfo, REG_ITMP1);
4293         x86_64_call_reg(cd, REG_ITMP1);
4294
4295         M_AST(REG_RESULT, REG_SP, (stackframesize - 3) * 8);
4296         M_ALD(REG_ITMP2, REG_RESULT, 0);
4297         M_AST(REG_ITMP2, REG_SP, (stackframesize - 4) * 8);
4298         M_MOV(REG_SP, REG_ITMP2);
4299         M_AADD_IMM((stackframesize - 4) * 8, REG_ITMP2);
4300         M_AST(REG_ITMP2, REG_RESULT, 0);
4301
4302         STATS({
4303                 x86_64_mov_imm_reg(cd, (ptrint) nativeinvokation, REG_ITMP1);
4304                 x86_64_call_reg(cd, REG_ITMP1);
4305         })
4306
4307         /* restore integer and float argument registers */
4308
4309         for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4310                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4311                         M_LLD(rd->argintregs[j++], REG_SP, i * 8);
4312
4313         for (i = 0, j = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4314                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4315                         M_DLD(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4316
4317
4318         /* copy or spill arguments to new locations */
4319
4320         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
4321                 t = md->paramtypes[i].type;
4322
4323                 if (IS_INT_LNG_TYPE(t)) {
4324                         if (!md->params[i].inmemory) {
4325                                 s1 = rd->argintregs[md->params[i].regoff];
4326
4327                                 if (!nmd->params[j].inmemory) {
4328                                         s2 = rd->argintregs[nmd->params[j].regoff];
4329                                         M_INTMOVE(s1, s2);
4330
4331                                 } else {
4332                                         s2 = nmd->params[j].regoff;
4333                                         M_LST(s1, REG_SP, s2 * 8);
4334                                 }
4335
4336                         } else {
4337                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4338                                 s2 = nmd->params[j].regoff;
4339                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
4340                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
4341                         }
4342
4343                 } else {
4344                         /* We only copy spilled float arguments, as the float argument    */
4345                         /* registers keep unchanged.                                      */
4346
4347                         if (md->params[i].inmemory) {
4348                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4349                                 s2 = nmd->params[j].regoff;
4350                                 M_DLD(REG_FTMP1, REG_SP, s1 * 8);
4351                                 M_DST(REG_FTMP1, REG_SP, s2 * 8);
4352                         }
4353                 }
4354         }
4355
4356         /* put class into second argument register */
4357
4358         if (m->flags & ACC_STATIC)
4359                 x86_64_mov_imm_reg(cd, (ptrint) m->class, rd->argintregs[1]);
4360
4361         /* put env into first argument register */
4362
4363         x86_64_mov_imm_reg(cd, (ptrint) &env, rd->argintregs[0]);
4364
4365         /* do the native function call */
4366
4367 #if !defined(STATIC_CLASSPATH)
4368         if (f == NULL) {
4369                 codegen_addpatchref(cd, cd->mcodeptr, PATCHER_resolve_native, m);
4370
4371                 if (showdisassemble) {
4372                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
4373                 }
4374         }
4375 #endif
4376
4377         x86_64_mov_imm_reg(cd, (ptrint) f, REG_ITMP1);
4378         x86_64_call_reg(cd, REG_ITMP1);
4379
4380
4381         /* remove dynamic stack info */
4382
4383         M_LLD(REG_ITMP2, REG_SP, (stackframesize - 4) * 8);
4384         M_LLD(REG_ITMP3, REG_SP, (stackframesize - 3) * 8);
4385         M_LST(REG_ITMP2, REG_ITMP3, 0);
4386
4387
4388         /* generate call trace */
4389
4390         if (runverbose) {
4391                 M_LST(REG_RESULT, REG_SP, 0 * 8);
4392                 M_DST(REG_FRESULT, REG_SP, 1 * 8);
4393
4394                 x86_64_mov_imm_reg(cd, (ptrint) m, rd->argintregs[0]);
4395                 M_MOV(REG_RESULT, rd->argintregs[1]);
4396                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4397                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4398
4399                 x86_64_mov_imm_reg(cd, (ptrint) builtin_displaymethodstop, REG_ITMP1);
4400                 x86_64_call_reg(cd, REG_ITMP1);
4401
4402                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
4403                 M_DLD(REG_FRESULT, REG_SP, 1 * 8);
4404         }
4405
4406         /* check for exception */
4407
4408 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4409         M_LST(REG_RESULT, REG_SP, 0 * 8);
4410         x86_64_mov_imm_reg(cd, (ptrint) builtin_get_exceptionptrptr, REG_ITMP3);
4411         x86_64_call_reg(cd, REG_ITMP3);
4412         M_LLD(REG_ITMP3, REG_RESULT, 0);
4413         M_LLD(REG_RESULT, REG_SP, 0 * 8);
4414 #else
4415         x86_64_mov_imm_reg(cd, (ptrint) &_exceptionptr, REG_ITMP3);
4416         x86_64_mov_membase_reg(cd, REG_ITMP3, 0, REG_ITMP3);
4417 #endif
4418         x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
4419         x86_64_jcc(cd, X86_64_CC_NE, 7 + 1);
4420
4421         /* remove stackframe */
4422
4423         M_AADD_IMM(stackframesize * 8, REG_SP);
4424
4425         x86_64_ret(cd);
4426
4427
4428         /* handle exception */
4429
4430 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4431         M_LST(REG_ITMP3, REG_SP, 0 * 8);
4432         x86_64_mov_imm_reg(cd, (ptrint) builtin_get_exceptionptrptr, REG_ITMP3);
4433         x86_64_call_reg(cd, REG_ITMP3);
4434         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
4435         M_LLD(REG_ITMP1_XPTR, REG_SP, 0 * 8);
4436 #else
4437         x86_64_mov_reg_reg(cd, REG_ITMP3, REG_ITMP1_XPTR);
4438         x86_64_mov_imm_reg(cd, (ptrint) &_exceptionptr, REG_ITMP3);
4439         x86_64_alu_reg_reg(cd, X86_64_XOR, REG_ITMP2, REG_ITMP2);
4440         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_ITMP3, 0);    /* clear exception pointer */
4441 #endif
4442
4443         /* remove stackframe */
4444
4445         M_AADD_IMM(stackframesize * 8, REG_SP);
4446
4447         M_LLD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
4448         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
4449
4450         x86_64_mov_imm_reg(cd, (ptrint) asm_handle_nat_exception, REG_ITMP3);
4451         x86_64_jmp_reg(cd, REG_ITMP3);
4452
4453
4454         /* process patcher calls **************************************************/
4455
4456         {
4457                 u1          *xcodeptr;
4458                 patchref    *pref;
4459                 codegendata *tmpcd;
4460                 ptrint       mcode;
4461
4462                 tmpcd = DNEW(codegendata);
4463
4464                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4465                         /* Get machine code which is patched back in later. A             */
4466                         /* `call rel32' is 5 bytes long (but read 8 bytes).               */
4467
4468                         xcodeptr = cd->mcodebase + pref->branchpos;
4469                         mcode = *((ptrint *) xcodeptr);
4470
4471                         /* patch in `call rel32' to call the following code               */
4472
4473                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
4474                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
4475
4476                         /* move pointer to java_objectheader onto stack */
4477
4478 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4479                         /* create a virtual java_objectheader */
4480
4481                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4482                         off = dseg_addaddress(cd, NULL);                    /* vftbl      */
4483
4484                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + off, REG_ITMP3);
4485                         x86_64_push_reg(cd, REG_ITMP3);
4486 #else
4487                         x86_64_push_imm(cd, 0);
4488 #endif
4489
4490                         /* move machine code bytes and classinfo pointer into registers */
4491
4492                         x86_64_mov_imm_reg(cd, (ptrint) mcode, REG_ITMP3);
4493                         x86_64_push_reg(cd, REG_ITMP3);
4494                         x86_64_mov_imm_reg(cd, (ptrint) pref->ref, REG_ITMP3);
4495                         x86_64_push_reg(cd, REG_ITMP3);
4496
4497                         x86_64_mov_imm_reg(cd, (ptrint) pref->patcher, REG_ITMP3);
4498                         x86_64_push_reg(cd, REG_ITMP3);
4499
4500                         x86_64_mov_imm_reg(cd, (ptrint) asm_wrapper_patcher, REG_ITMP3);
4501                         x86_64_jmp_reg(cd, REG_ITMP3);
4502                 }
4503         }
4504
4505         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
4506
4507         return m->entrypoint;
4508 }
4509
4510
4511 /*
4512  * These are local overrides for various environment variables in Emacs.
4513  * Please do not remove this and leave it at the end of the file, where
4514  * Emacs will automagically detect them.
4515  * ---------------------------------------------------------------------
4516  * Local variables:
4517  * mode: c
4518  * indent-tabs-mode: t
4519  * c-basic-offset: 4
4520  * tab-width: 4
4521  * End:
4522  */