src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005 R. Grafl, A. Krall, C. Kruegel, C. Oates,
   4    R. Obermaisser, M. Platter, M. Probst, S. Ring, E. Steiner,
   5    C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich, J. Wenninger,
   6    Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  23    02111-1307, USA.
  24
  25    Contact: cacao@complang.tuwien.ac.at
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes:
  31
  32    $Id: codegen.c 2656 2005-06-13 14:14:24Z twisti $
  33
  34 */
  35
  36
  37 #include <stdio.h>
  38
  39 #include "config.h"
  40
  41 #include "md.h"
  42 #include "md-abi.h"
  43 #include "md-abi.inc"
  44
  45 #include "vm/jit/x86_64/arch.h"
  46 #include "vm/jit/x86_64/codegen.h"
  47 #include "vm/jit/x86_64/emitfuncs.h"
  48 #include "vm/jit/x86_64/types.h"
  49 #include "vm/jit/x86_64/asmoffsets.h"
  50
  51 #include "cacao/cacao.h"
  52 #include "native/native.h"
  53 #include "vm/global.h"
  54 #include "vm/builtin.h"
  55 #include "vm/loader.h"
  56 #include "vm/statistics.h"
  57 #include "vm/stringlocal.h"
  58 #include "vm/tables.h"
  59 #include "vm/jit/asmpart.h"
  60 #include "vm/jit/codegen.inc"
  61 #include "vm/jit/helper.h"
  62 #include "vm/jit/jit.h"
  63
  64 #if defined(LSRA)
  65 # include "vm/jit/lsra.inc"
  66 #endif
  67
  68 #include "vm/jit/parse.h"
  69 #include "vm/jit/patcher.h"
  70 #include "vm/jit/reg.h"
  71 #include "vm/jit/reg.inc"
  72
  73
  74 /* codegen *********************************************************************
  75
  76    Generates machine code.
  77
  78 *******************************************************************************/
  79
  80 void codegen(methodinfo *m, codegendata *cd, registerdata *rd)
  81 {
  82         s4                  len, s1, s2, s3, d;
  83         u2                  currentline;
  84         ptrint              a;
  85         s4                  parentargs_base;
  86         stackptr            src;
  87         varinfo            *var;
  88         basicblock         *bptr;
  89         instruction        *iptr;
  90         exceptiontable     *ex;
  91         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
  92         builtintable_entry *bte;
  93         methoddesc         *md;
  94
  95         {
  96         s4 i, p, t, l;
  97         s4 savedregs_num;
  98
  99         savedregs_num = 0;
 100
 101         /* space to save used callee saved registers */
 102
 103         savedregs_num += (rd->savintregcnt - rd->maxsavintreguse);
 104         savedregs_num += (rd->savfltregcnt - rd->maxsavfltreguse);
 105
 106         parentargs_base = rd->maxmemuse + savedregs_num;
 107
 108 #if defined(USE_THREADS)           /* space to save argument of monitor_enter */
 109
 110         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 111                 parentargs_base++;
 112
 113 #endif
 114
 115     /* Keep stack of non-leaf functions 16-byte aligned for calls into native */
 116         /* code e.g. libc or jni (alignment problems with movaps).                */
 117
 118         if (!m->isleafmethod || runverbose)
 119                 parentargs_base |= 0x1;
 120
 121         /* create method header */
 122
 123         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 124         (void) dseg_adds4(cd, parentargs_base * 8);             /* FrameSize      */
 125
 126 #if defined(USE_THREADS)
 127
 128         /* IsSync contains the offset relative to the stack pointer for the
 129            argument of monitor_exit used in the exception handler. Since the
 130            offset could be zero and give a wrong meaning of the flag it is
 131            offset by one.
 132         */
 133
 134         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 135                 (void) dseg_adds4(cd, (rd->maxmemuse + 1) * 8);     /* IsSync         */
 136         else
 137
 138 #endif
 139
 140                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 141
 142         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 143         (void) dseg_adds4(cd, rd->savintregcnt - rd->maxsavintreguse);/* IntSave  */
 144         (void) dseg_adds4(cd, rd->savfltregcnt - rd->maxsavfltreguse);/* FltSave  */
 145
 146         (void) dseg_addlinenumbertablesize(cd);
 147
 148         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 149
 150         /* create exception table */
 151
 152         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 153                 dseg_addtarget(cd, ex->start);
 154                 dseg_addtarget(cd, ex->end);
 155                 dseg_addtarget(cd, ex->handler);
 156                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 157         }
 158
 159         /* initialize mcode variables */
 160
 161         cd->mcodeptr = (u1 *) cd->mcodebase;
 162         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
 163         MCODECHECK(128 + m->paramcount);
 164
 165         /* create stack frame (if necessary) */
 166
 167         if (parentargs_base) {
 168                 x86_64_alu_imm_reg(cd, X86_64_SUB, parentargs_base * 8, REG_SP);
 169         }
 170
 171         /* save used callee saved registers */
 172
 173         p = parentargs_base;
 174         for (i = rd->savintregcnt - 1; i >= rd->maxsavintreguse; i--) {
 175                 p--; x86_64_mov_reg_membase(cd, rd->savintregs[i], REG_SP, p * 8);
 176         }
 177         for (i = rd->savfltregcnt - 1; i >= rd->maxsavfltreguse; i--) {
 178                 p--; x86_64_movq_reg_membase(cd, rd->savfltregs[i], REG_SP, p * 8);
 179         }
 180
 181         /* take arguments out of register or stack frame */
 182
 183         md = m->parseddesc;
 184
 185         for (p = 0, l = 0; p < md->paramcount; p++) {
 186                 t = md->paramtypes[p].type;
 187                 var = &(rd->locals[l][t]);
 188                 l++;
 189                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 190                         l++;
 191                 if (var->type < 0)
 192                         continue;
 193                 s1 = md->params[p].regoff;
 194                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 195                         s2 = rd->argintregs[s1];
 196                         if (!md->params[p].inmemory) {           /* register arguments    */
 197                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 198                                         M_INTMOVE(s2, var->regoff);
 199
 200                                 } else {                             /* reg arg -> spilled    */
 201                                     M_LST(s2, REG_SP, var->regoff * 8);
 202                                 }
 203
 204                         } else {                                 /* stack arguments       */
 205                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 206                                         /* + 8 for return address */
 207                                         M_LLD(var->regoff, REG_SP, (parentargs_base + s1) * 8 + 8);
 208
 209                                 } else {                             /* stack arg -> spilled  */
 210                                         var->regoff = parentargs_base + s1 + 1;
 211                                 }
 212                         }
 213
 214                 } else {                                     /* floating args         */
 215                         if (!md->params[p].inmemory) {           /* register arguments    */
 216                                 s2 = rd->argfltregs[s1];
 217                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 218                                         M_FLTMOVE(s2, var->regoff);
 219
 220                                 } else {                                         /* reg arg -> spilled    */
 221                                         M_DST(s2, REG_SP, var->regoff * 8);
 222                                 }
 223
 224                         } else {                                 /* stack arguments       */
 225                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 226                                         M_DLD(var->regoff, REG_SP, (parentargs_base + s1) * 8 + 8);
 227
 228                                 } else {
 229                                         var->regoff = parentargs_base + s1 + 1;
 230                                 }
 231                         }
 232                 }
 233         }  /* end for */
 234
 235         /* save monitorenter argument */
 236
 237 #if defined(USE_THREADS)
 238         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 239                 /* stack offset for monitor argument */
 240
 241                 s1 = rd->maxmemuse;
 242
 243                 if (runverbose) {
 244                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 245
 246                         for (p = 0; p < INT_ARG_CNT; p++)
 247                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 248
 249                         for (p = 0; p < FLT_ARG_CNT; p++)
 250                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 251
 252                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 253                 }
 254
 255                 /* decide which monitor enter function to call */
 256
 257                 if (m->flags & ACC_STATIC) {
 258                         x86_64_mov_imm_reg(cd, (ptrint) m->class, REG_ITMP1);
 259                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, s1 * 8);
 260                         M_INTMOVE(REG_ITMP1, rd->argintregs[0]);
 261                         x86_64_mov_imm_reg(cd, (ptrint) BUILTIN_staticmonitorenter, REG_ITMP1);
 262                         x86_64_call_reg(cd, REG_ITMP1);
 263
 264                 } else {
 265                         x86_64_test_reg_reg(cd, rd->argintregs[0], rd->argintregs[0]);
 266                         x86_64_jcc(cd, X86_64_CC_Z, 0);
 267                         codegen_addxnullrefs(cd, cd->mcodeptr);
 268                         x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, s1 * 8);
 269                         x86_64_mov_imm_reg(cd, (ptrint) BUILTIN_monitorenter, REG_ITMP1);
 270                         x86_64_call_reg(cd, REG_ITMP1);
 271                 }
 272
 273                 if (runverbose) {
 274                         for (p = 0; p < INT_ARG_CNT; p++)
 275                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 276
 277                         for (p = 0; p < FLT_ARG_CNT; p++)
 278                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 279
 280                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 281                 }
 282         }
 283 #endif
 284
 285         /* Copy argument registers to stack and call trace function with pointer  */
 286         /* to arguments on stack.                                                 */
 287
 288         if (runverbose || opt_stat) {
 289                 M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 290
 291                 /* save integer argument registers */
 292
 293                 for (p = 0; p < INT_ARG_CNT; p++)
 294                         M_LST(rd->argintregs[p], REG_SP, (1 + p) * 8);
 295
 296                 /* save float argument registers */
 297
 298                 for (p = 0; p < FLT_ARG_CNT; p++)
 299                         M_DST(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 300
 301                 /* save temporary registers for leaf methods */
 302
 303                 if (m->isleafmethod) {
 304                         for (p = 0; p < INT_TMP_CNT; p++)
 305                                 M_LST(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 306
 307                         for (p = 0; p < FLT_TMP_CNT; p++)
 308                                 M_DST(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 309                 }
 310
 311                 if (runverbose) {
 312                         /* show integer hex code for float arguments */
 313
 314                         for (p = 0, l = 0; p < m->paramcount && p < INT_ARG_CNT; p++) {
 315                                 t = m->paramtypes[p];
 316
 317                                 /* if the paramtype is a float, we have to right shift all        */
 318                                 /* following integer registers                                    */
 319
 320                                 if (IS_FLT_DBL_TYPE(t)) {
 321                                         for (s1 = INT_ARG_CNT - 2; s1 >= p; s1--) {
 322                                                 x86_64_mov_reg_reg(cd, rd->argintregs[s1], rd->argintregs[s1 + 1]);
 323                                         }
 324
 325                                         x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 326                                         l++;
 327                                 }
 328                         }
 329
 330                         x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP2);
 331                         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_SP, 0 * 8);
 332                         x86_64_mov_imm_reg(cd, (ptrint) builtin_trace_args, REG_ITMP1);
 333                         x86_64_call_reg(cd, REG_ITMP1);
 334                 }
 335                 STATS({
 336                         x86_64_mov_imm_reg(cd,(u8)compiledinvokation,REG_ITMP1);
 337                         x86_64_call_reg(cd,REG_ITMP1);
 338                 })
 339
 340                 /* restore integer argument registers */
 341
 342                 for (p = 0; p < INT_ARG_CNT; p++)
 343                         M_LLD(rd->argintregs[p], REG_SP, (1 + p) * 8);
 344
 345                 /* restore float argument registers */
 346
 347                 for (p = 0; p < FLT_ARG_CNT; p++)
 348                         M_DLD(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 349
 350                 /* restore temporary registers for leaf methods */
 351
 352                 if (m->isleafmethod) {
 353                         for (p = 0; p < INT_TMP_CNT; p++)
 354                                 M_LLD(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 355
 356                         for (p = 0; p < FLT_TMP_CNT; p++)
 357                                 M_DLD(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 358                 }
 359
 360                 M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 361         }
 362
 363         }
 364
 365         /* end of header generation */
 366
 367         /* walk through all basic blocks */
 368         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 369
 370                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 371
 372                 if (bptr->flags >= BBREACHED) {
 373
 374                         /* branch resolving */
 375
 376                         branchref *bref;
 377                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 378                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 379                                                                   bref->branchpos,
 380                                                                   bptr->mpc);
 381                         }
 382
 383                 /* copy interface registers to their destination */
 384
 385                 src = bptr->instack;
 386                 len = bptr->indepth;
 387                 MCODECHECK(64 + len);
 388
 389 #ifdef LSRA
 390                 if (opt_lsra) {
 391                         while (src != NULL) {
 392                                 len--;
 393                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 394                                         if (bptr->type == BBTYPE_SBR) {
 395                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 396                                                 if (!(src->flags & INMEMORY))
 397                                                         d= src->regoff;
 398                                                 else
 399                                                         d=REG_ITMP1;
 400                                                 x86_64_pop_reg(cd, d);
 401                                                 store_reg_to_var_int(src, d);
 402
 403                                         } else if (bptr->type == BBTYPE_EXH) {
 404                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 405                                                 if (!(src->flags & INMEMORY))
 406                                                         d= src->regoff;
 407                                                 else
 408                                                         d=REG_ITMP1;
 409                                                 M_INTMOVE(REG_ITMP1, d);
 410                                                 store_reg_to_var_int(src, d);
 411                                         }
 412                                 }
 413                                 src = src->prev;
 414                         }
 415
 416                 } else {
 417 #endif
 418
 419         while (src != NULL) {
 420                         len--;
 421                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 422                                 if (bptr->type == BBTYPE_SBR) {
 423                                         d = reg_of_var(rd, src, REG_ITMP1);
 424                                         x86_64_pop_reg(cd, d);
 425                                         store_reg_to_var_int(src, d);
 426
 427                                 } else if (bptr->type == BBTYPE_EXH) {
 428                                         d = reg_of_var(rd, src, REG_ITMP1);
 429                                         M_INTMOVE(REG_ITMP1, d);
 430                                         store_reg_to_var_int(src, d);
 431                                 }
 432
 433                         } else {
 434                                 d = reg_of_var(rd, src, REG_ITMP1);
 435                                 if ((src->varkind != STACKVAR)) {
 436                                         s2 = src->type;
 437                                         if (IS_FLT_DBL_TYPE(s2)) {
 438                                                 s1 = rd->interfaces[len][s2].regoff;
 439                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 440                                                         M_FLTMOVE(s1, d);
 441
 442                                                 } else {
 443                                                         x86_64_movq_membase_reg(cd, REG_SP, s1 * 8, d);
 444                                                 }
 445                                                 store_reg_to_var_flt(src, d);
 446
 447                                         } else {
 448                                                 s1 = rd->interfaces[len][s2].regoff;
 449                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 450                                                         M_INTMOVE(s1, d);
 451
 452                                                 } else {
 453                                                         x86_64_mov_membase_reg(cd, REG_SP, s1 * 8, d);
 454                                                 }
 455                                                 store_reg_to_var_int(src, d);
 456                                         }
 457                                 }
 458                         }
 459                         src = src->prev;
 460                 }
 461 #ifdef LSRA
 462                 }
 463 #endif
 464                 /* walk through all instructions */
 465
 466                 src = bptr->instack;
 467                 len = bptr->icount;
 468                 currentline=0;
 469                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 470                         if (iptr->line != currentline) {
 471                                 dseg_addlinenumber(cd, iptr->line, cd->mcodeptr);
 472                                 /*printf("%s : %d\n",m->name->text,iptr->line);*/
 473                                 currentline = iptr->line;
 474                         }
 475
 476                         MCODECHECK(128);    /* XXX are 128 bytes enough? */
 477
 478                         switch (iptr->opc) {
 479                         case ICMD_INLINE_START: /* internal ICMDs                         */
 480                         case ICMD_INLINE_END:
 481                                 break;
 482
 483                         case ICMD_NOP:    /* ...  ==> ...                                 */
 484                                 break;
 485
 486                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 487                                 if (src->flags & INMEMORY) {
 488                                         x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
 489
 490                                 } else {
 491                                         x86_64_test_reg_reg(cd, src->regoff, src->regoff);
 492                                 }
 493                                 x86_64_jcc(cd, X86_64_CC_Z, 0);
 494                                 codegen_addxnullrefs(cd, cd->mcodeptr);
 495                                 break;
 496
 497                 /* constant operations ************************************************/
 498
 499                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 500                                       /* op1 = 0, val.i = constant                    */
 501
 502                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 503                         if (iptr->val.i == 0) {
 504                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 505                         } else {
 506                                 x86_64_movl_imm_reg(cd, iptr->val.i, d);
 507                         }
 508                         store_reg_to_var_int(iptr->dst, d);
 509                         break;
 510
 511                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 512                                       /* op1 = 0, val.a = constant                    */
 513
 514                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 515                         if (iptr->val.a == 0) {
 516                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 517                         } else {
 518                                 x86_64_mov_imm_reg(cd, (s8) iptr->val.a, d);
 519                         }
 520                         store_reg_to_var_int(iptr->dst, d);
 521                         break;
 522
 523                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 524                                       /* op1 = 0, val.l = constant                    */
 525
 526                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 527                         if (iptr->val.l == 0) {
 528                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 529                         } else {
 530                                 x86_64_mov_imm_reg(cd, iptr->val.l, d);
 531                         }
 532                         store_reg_to_var_int(iptr->dst, d);
 533                         break;
 534
 535                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 536                                       /* op1 = 0, val.f = constant                    */
 537
 538                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 539                         a = dseg_addfloat(cd, iptr->val.f);
 540                         x86_64_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + a, d);
 541                         store_reg_to_var_flt(iptr->dst, d);
 542                         break;
 543
 544                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 545                                       /* op1 = 0, val.d = constant                    */
 546
 547                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 548                         a = dseg_adddouble(cd, iptr->val.d);
 549                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, d);
 550                         store_reg_to_var_flt(iptr->dst, d);
 551                         break;
 552
 553
 554                 /* load/store operations **********************************************/
 555
 556                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 557                                       /* op1 = local variable                         */
 558
 559                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 560                         if ((iptr->dst->varkind == LOCALVAR) &&
 561                             (iptr->dst->varnum == iptr->op1)) {
 562                                 break;
 563                         }
 564                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 565                         if (var->flags & INMEMORY) {
 566                                 x86_64_movl_membase_reg(cd, REG_SP, var->regoff * 8, d);
 567                                 store_reg_to_var_int(iptr->dst, d);
 568
 569                         } else {
 570                                 if (iptr->dst->flags & INMEMORY) {
 571                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 572
 573                                 } else {
 574                                         M_INTMOVE(var->regoff, d);
 575                                 }
 576                         }
 577                         break;
 578
 579                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 580                 case ICMD_ALOAD:      /* op1 = local variable                         */
 581
 582                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 583                         if ((iptr->dst->varkind == LOCALVAR) &&
 584                             (iptr->dst->varnum == iptr->op1)) {
 585                                 break;
 586                         }
 587                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 588                         if (var->flags & INMEMORY) {
 589                                 x86_64_mov_membase_reg(cd, REG_SP, var->regoff * 8, d);
 590                                 store_reg_to_var_int(iptr->dst, d);
 591
 592                         } else {
 593                                 if (iptr->dst->flags & INMEMORY) {
 594                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 595
 596                                 } else {
 597                                         M_INTMOVE(var->regoff, d);
 598                                 }
 599                         }
 600                         break;
 601
 602                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 603                 case ICMD_DLOAD:      /* op1 = local variable                         */
 604
 605                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 606                         if ((iptr->dst->varkind == LOCALVAR) &&
 607                             (iptr->dst->varnum == iptr->op1)) {
 608                                 break;
 609                         }
 610                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 611                         if (var->flags & INMEMORY) {
 612                                 x86_64_movq_membase_reg(cd, REG_SP, var->regoff * 8, d);
 613                                 store_reg_to_var_flt(iptr->dst, d);
 614
 615                         } else {
 616                                 if (iptr->dst->flags & INMEMORY) {
 617                                         x86_64_movq_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 618
 619                                 } else {
 620                                         M_FLTMOVE(var->regoff, d);
 621                                 }
 622                         }
 623                         break;
 624
 625                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 626                 case ICMD_LSTORE:     /* op1 = local variable                         */
 627                 case ICMD_ASTORE:
 628
 629                         if ((src->varkind == LOCALVAR) &&
 630                             (src->varnum == iptr->op1)) {
 631                                 break;
 632                         }
 633                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 634                         if (var->flags & INMEMORY) {
 635                                 var_to_reg_int(s1, src, REG_ITMP1);
 636                                 x86_64_mov_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 637
 638                         } else {
 639                                 var_to_reg_int(s1, src, var->regoff);
 640                                 M_INTMOVE(s1, var->regoff);
 641                         }
 642                         break;
 643
 644                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 645                 case ICMD_DSTORE:     /* op1 = local variable                         */
 646
 647                         if ((src->varkind == LOCALVAR) &&
 648                             (src->varnum == iptr->op1)) {
 649                                 break;
 650                         }
 651                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 652                         if (var->flags & INMEMORY) {
 653                                 var_to_reg_flt(s1, src, REG_FTMP1);
 654                                 x86_64_movq_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 655
 656                         } else {
 657                                 var_to_reg_flt(s1, src, var->regoff);
 658                                 M_FLTMOVE(s1, var->regoff);
 659                         }
 660                         break;
 661
 662
 663                 /* pop/dup/swap operations ********************************************/
 664
 665                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 666
 667                 case ICMD_POP:        /* ..., value  ==> ...                          */
 668                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 669                         break;
 670
 671                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 672                         M_COPY(src, iptr->dst);
 673                         break;
 674
 675                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 676
 677                         M_COPY(src,       iptr->dst);
 678                         M_COPY(src->prev, iptr->dst->prev);
 679                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 680                         break;
 681
 682                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 683
 684                         M_COPY(src,             iptr->dst);
 685                         M_COPY(src->prev,       iptr->dst->prev);
 686                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 687                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 688                         break;
 689
 690                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 691
 692                         M_COPY(src,       iptr->dst);
 693                         M_COPY(src->prev, iptr->dst->prev);
 694                         break;
 695
 696                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 697
 698                         M_COPY(src,             iptr->dst);
 699                         M_COPY(src->prev,       iptr->dst->prev);
 700                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 701                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 702                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 703                         break;
 704
 705                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 706
 707                         M_COPY(src,                   iptr->dst);
 708                         M_COPY(src->prev,             iptr->dst->prev);
 709                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 710                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 711                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 712                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 713                         break;
 714
 715                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 716
 717                         M_COPY(src,       iptr->dst->prev);
 718                         M_COPY(src->prev, iptr->dst);
 719                         break;
 720
 721
 722                 /* integer operations *************************************************/
 723
 724                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 725
 726                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 727                         if (iptr->dst->flags & INMEMORY) {
 728                                 if (src->flags & INMEMORY) {
 729                                         if (src->regoff == iptr->dst->regoff) {
 730                                                 x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 731
 732                                         } else {
 733                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 734                                                 x86_64_negl_reg(cd, REG_ITMP1);
 735                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 736                                         }
 737
 738                                 } else {
 739                                         x86_64_movl_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 740                                         x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 741                                 }
 742
 743                         } else {
 744                                 if (src->flags & INMEMORY) {
 745                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 746                                         x86_64_negl_reg(cd, d);
 747
 748                                 } else {
 749                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 750                                         x86_64_negl_reg(cd, iptr->dst->regoff);
 751                                 }
 752                         }
 753                         break;
 754
 755                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 756
 757                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 758                         if (iptr->dst->flags & INMEMORY) {
 759                                 if (src->flags & INMEMORY) {
 760                                         if (src->regoff == iptr->dst->regoff) {
 761                                                 x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 762
 763                                         } else {
 764                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 765                                                 x86_64_neg_reg(cd, REG_ITMP1);
 766                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 767                                         }
 768
 769                                 } else {
 770                                         x86_64_mov_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 771                                         x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 772                                 }
 773
 774                         } else {
 775                                 if (src->flags & INMEMORY) {
 776                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 777                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 778
 779                                 } else {
 780                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 781                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 782                                 }
 783                         }
 784                         break;
 785
 786                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 787
 788                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 789                         if (src->flags & INMEMORY) {
 790                                 x86_64_movslq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 791
 792                         } else {
 793                                 x86_64_movslq_reg_reg(cd, src->regoff, d);
 794                         }
 795                         store_reg_to_var_int(iptr->dst, d);
 796                         break;
 797
 798                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 799
 800                         var_to_reg_int(s1, src, REG_ITMP1);
 801                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 802                         M_INTMOVE(s1, d);
 803                         store_reg_to_var_int(iptr->dst, d);
 804                         break;
 805
 806                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 807
 808                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 809                         if (src->flags & INMEMORY) {
 810                                 x86_64_movsbq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 811
 812                         } else {
 813                                 x86_64_movsbq_reg_reg(cd, src->regoff, d);
 814                         }
 815                         store_reg_to_var_int(iptr->dst, d);
 816                         break;
 817
 818                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 819
 820                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 821                         if (src->flags & INMEMORY) {
 822                                 x86_64_movzwq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 823
 824                         } else {
 825                                 x86_64_movzwq_reg_reg(cd, src->regoff, d);
 826                         }
 827                         store_reg_to_var_int(iptr->dst, d);
 828                         break;
 829
 830                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 831
 832                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 833                         if (src->flags & INMEMORY) {
 834                                 x86_64_movswq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 835
 836                         } else {
 837                                 x86_64_movswq_reg_reg(cd, src->regoff, d);
 838                         }
 839                         store_reg_to_var_int(iptr->dst, d);
 840                         break;
 841
 842
 843                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 844
 845                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 846                         x86_64_emit_ialu(cd, X86_64_ADD, src, iptr);
 847                         break;
 848
 849                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 850                                       /* val.i = constant                             */
 851
 852                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 853                         x86_64_emit_ialuconst(cd, X86_64_ADD, src, iptr);
 854                         break;
 855
 856                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 857
 858                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 859                         x86_64_emit_lalu(cd, X86_64_ADD, src, iptr);
 860                         break;
 861
 862                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 863                                       /* val.l = constant                             */
 864
 865                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 866                         x86_64_emit_laluconst(cd, X86_64_ADD, src, iptr);
 867                         break;
 868
 869                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 870
 871                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 872                         if (iptr->dst->flags & INMEMORY) {
 873                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 874                                         if (src->prev->regoff == iptr->dst->regoff) {
 875                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 876                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 877
 878                                         } else {
 879                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 880                                                 x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 881                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 882                                         }
 883
 884                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 885                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 886                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 887                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 888
 889                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 890                                         if (src->prev->regoff == iptr->dst->regoff) {
 891                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 892
 893                                         } else {
 894                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 895                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 896                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 897                                         }
 898
 899                                 } else {
 900                                         x86_64_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 901                                         x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 902                                 }
 903
 904                         } else {
 905                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 906                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 907                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 908
 909                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 910                                         M_INTMOVE(src->prev->regoff, d);
 911                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 912
 913                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 914                                         /* workaround for reg alloc */
 915                                         if (src->regoff == iptr->dst->regoff) {
 916                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 917                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 918                                                 M_INTMOVE(REG_ITMP1, d);
 919
 920                                         } else {
 921                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 922                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 923                                         }
 924
 925                                 } else {
 926                                         /* workaround for reg alloc */
 927                                         if (src->regoff == iptr->dst->regoff) {
 928                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
 929                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 930                                                 M_INTMOVE(REG_ITMP1, d);
 931
 932                                         } else {
 933                                                 M_INTMOVE(src->prev->regoff, d);
 934                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 935                                         }
 936                                 }
 937                         }
 938                         break;
 939
 940                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 941                                       /* val.i = constant                             */
 942
 943                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 944                         x86_64_emit_ialuconst(cd, X86_64_SUB, src, iptr);
 945                         break;
 946
 947                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 948
 949                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 950                         if (iptr->dst->flags & INMEMORY) {
 951                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 952                                         if (src->prev->regoff == iptr->dst->regoff) {
 953                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 954                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 955
 956                                         } else {
 957                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 958                                                 x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 959                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 960                                         }
 961
 962                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 963                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 964                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 965                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 966
 967                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 968                                         if (src->prev->regoff == iptr->dst->regoff) {
 969                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 970
 971                                         } else {
 972                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 973                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 974                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 975                                         }
 976
 977                                 } else {
 978                                         x86_64_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 979                                         x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 980                                 }
 981
 982                         } else {
 983                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 984                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 985                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 986
 987                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 988                                         M_INTMOVE(src->prev->regoff, d);
 989                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 990
 991                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 992                                         /* workaround for reg alloc */
 993                                         if (src->regoff == iptr->dst->regoff) {
 994                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 995                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 996                                                 M_INTMOVE(REG_ITMP1, d);
 997
 998                                         } else {
 999                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1000                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1001                                         }
1002
1003                                 } else {
1004                                         /* workaround for reg alloc */
1005                                         if (src->regoff == iptr->dst->regoff) {
1006                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1007                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1008                                                 M_INTMOVE(REG_ITMP1, d);
1009
1010                                         } else {
1011                                                 M_INTMOVE(src->prev->regoff, d);
1012                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1013                                         }
1014                                 }
1015                         }
1016                         break;
1017
1018                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1019                                       /* val.l = constant                             */
1020
1021                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1022                         x86_64_emit_laluconst(cd, X86_64_SUB, src, iptr);
1023                         break;
1024
1025                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1026
1027                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1028                         if (iptr->dst->flags & INMEMORY) {
1029                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1030                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1031                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1032                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1033
1034                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1035                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1036                                         x86_64_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1037                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1038
1039                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1040                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1041                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1042                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1043
1044                                 } else {
1045                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1046                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1047                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1048                                 }
1049
1050                         } else {
1051                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1052                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1053                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1054
1055                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1056                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1057                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1058
1059                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1060                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1061                                         x86_64_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1062
1063                                 } else {
1064                                         if (src->regoff == iptr->dst->regoff) {
1065                                                 x86_64_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1066
1067                                         } else {
1068                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1069                                                 x86_64_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1070                                         }
1071                                 }
1072                         }
1073                         break;
1074
1075                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1076                                       /* val.i = constant                             */
1077
1078                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1079                         if (iptr->dst->flags & INMEMORY) {
1080                                 if (src->flags & INMEMORY) {
1081                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1082                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1083
1084                                 } else {
1085                                         x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1086                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1087                                 }
1088
1089                         } else {
1090                                 if (src->flags & INMEMORY) {
1091                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1092
1093                                 } else {
1094                                         if (iptr->val.i == 2) {
1095                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1096                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1097
1098                                         } else {
1099                                                 x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1100                                         }
1101                                 }
1102                         }
1103                         break;
1104
1105                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1106
1107                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1108                         if (iptr->dst->flags & INMEMORY) {
1109                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1110                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1111                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1112                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1113
1114                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1115                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1116                                         x86_64_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1117                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1118
1119                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1120                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1121                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1122                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1123
1124                                 } else {
1125                                         x86_64_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1126                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1127                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1128                                 }
1129
1130                         } else {
1131                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1132                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1133                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1134
1135                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1136                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1137                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1138
1139                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1140                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1141                                         x86_64_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1142
1143                                 } else {
1144                                         if (src->regoff == iptr->dst->regoff) {
1145                                                 x86_64_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1146
1147                                         } else {
1148                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1149                                                 x86_64_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1150                                         }
1151                                 }
1152                         }
1153                         break;
1154
1155                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1156                                       /* val.l = constant                             */
1157
1158                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1159                         if (iptr->dst->flags & INMEMORY) {
1160                                 if (src->flags & INMEMORY) {
1161                                         if (IS_IMM32(iptr->val.l)) {
1162                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1163
1164                                         } else {
1165                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1166                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1167                                         }
1168                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1169
1170                                 } else {
1171                                         if (IS_IMM32(iptr->val.l)) {
1172                                                 x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1173
1174                                         } else {
1175                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1176                                                 x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1177                                         }
1178                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1179                                 }
1180
1181                         } else {
1182                                 if (src->flags & INMEMORY) {
1183                                         if (IS_IMM32(iptr->val.l)) {
1184                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1185
1186                                         } else {
1187                                                 x86_64_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1188                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1189                                         }
1190
1191                                 } else {
1192                                         /* should match in many cases */
1193                                         if (iptr->val.l == 2) {
1194                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1195                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1196
1197                                         } else {
1198                                                 if (IS_IMM32(iptr->val.l)) {
1199                                                         x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1200
1201                                                 } else {
1202                                                         x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1203                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1204                                                         x86_64_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1205                                                 }
1206                                         }
1207                                 }
1208                         }
1209                         break;
1210
1211                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1212
1213                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1214                 if (src->prev->flags & INMEMORY) {
1215                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1216
1217                         } else {
1218                                 M_INTMOVE(src->prev->regoff, RAX);
1219                         }
1220
1221                         if (src->flags & INMEMORY) {
1222                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1223
1224                         } else {
1225                                 M_INTMOVE(src->regoff, REG_ITMP3);
1226                         }
1227                         gen_div_check(src);
1228
1229                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX);    /* check as described in jvm spec */
1230                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1231                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);      /* 4 bytes */
1232                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                  /* 6 bytes */
1233
1234                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1235                         x86_64_cltd(cd);
1236                         x86_64_idivl_reg(cd, REG_ITMP3);
1237
1238                         if (iptr->dst->flags & INMEMORY) {
1239                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1240                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1241
1242                         } else {
1243                                 M_INTMOVE(RAX, iptr->dst->regoff);
1244
1245                                 if (iptr->dst->regoff != RDX) {
1246                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1247                                 }
1248                         }
1249                         break;
1250
1251                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1252                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1253                         if (src->prev->flags & INMEMORY) {
1254                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1255
1256                         } else {
1257                                 M_INTMOVE(src->prev->regoff, RAX);
1258                         }
1259
1260                         if (src->flags & INMEMORY) {
1261                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1262
1263                         } else {
1264                                 M_INTMOVE(src->regoff, REG_ITMP3);
1265                         }
1266                         gen_div_check(src);
1267
1268                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1269
1270                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX);    /* check as described in jvm spec */
1271                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1272
1273
1274                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);           /* 2 bytes */
1275                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);      /* 4 bytes */
1276                         x86_64_jcc(cd, X86_64_CC_E, 1 + 3);                      /* 6 bytes */
1277
1278                         x86_64_cltd(cd);
1279                         x86_64_idivl_reg(cd, REG_ITMP3);
1280
1281                         if (iptr->dst->flags & INMEMORY) {
1282                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1283                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1284
1285                         } else {
1286                                 M_INTMOVE(RDX, iptr->dst->regoff);
1287
1288                                 if (iptr->dst->regoff != RDX) {
1289                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1290                                 }
1291                         }
1292                         break;
1293
1294                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1295                                       /* val.i = constant                             */
1296
1297                         var_to_reg_int(s1, src, REG_ITMP1);
1298                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1299                         M_INTMOVE(s1, REG_ITMP1);
1300                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1301                         x86_64_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1302                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1303                         x86_64_shiftl_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1304                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1305                         store_reg_to_var_int(iptr->dst, d);
1306                         break;
1307
1308                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1309                                       /* val.i = constant                             */
1310
1311                         var_to_reg_int(s1, src, REG_ITMP1);
1312                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1313                         M_INTMOVE(s1, REG_ITMP1);
1314                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1315                         x86_64_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1316                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1317                         x86_64_alul_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1318                         x86_64_alul_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1319                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1320                         store_reg_to_var_int(iptr->dst, d);
1321                         break;
1322
1323
1324                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1325
1326                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1327                 if (src->prev->flags & INMEMORY) {
1328                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1329
1330                         } else {
1331                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1332                         }
1333
1334                         if (src->flags & INMEMORY) {
1335                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1336
1337                         } else {
1338                                 M_INTMOVE(src->regoff, REG_ITMP3);
1339                         }
1340                         gen_div_check(src);
1341
1342                         x86_64_mov_imm_reg(cd, 0x8000000000000000LL, REG_ITMP2);    /* check as described in jvm spec */
1343                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
1344                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1345                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);          /* 4 bytes */
1346                         x86_64_jcc(cd, X86_64_CC_E, 3 + 2 + 3);                     /* 6 bytes */
1347
1348                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1349                         x86_64_cqto(cd);
1350                         x86_64_idiv_reg(cd, REG_ITMP3);
1351
1352                         if (iptr->dst->flags & INMEMORY) {
1353                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1354                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1355
1356                         } else {
1357                                 M_INTMOVE(RAX, iptr->dst->regoff);
1358
1359                                 if (iptr->dst->regoff != RDX) {
1360                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1361                                 }
1362                         }
1363                         break;
1364
1365                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1366
1367                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1368                         if (src->prev->flags & INMEMORY) {
1369                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1370
1371                         } else {
1372                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1373                         }
1374
1375                         if (src->flags & INMEMORY) {
1376                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1377
1378                         } else {
1379                                 M_INTMOVE(src->regoff, REG_ITMP3);
1380                         }
1381                         gen_div_check(src);
1382
1383                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1384
1385                         x86_64_mov_imm_reg(cd, 0x8000000000000000LL, REG_ITMP2);    /* check as described in jvm spec */
1386                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
1387                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1388
1389
1390                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);              /* 2 bytes */
1391                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);          /* 4 bytes */
1392                         x86_64_jcc(cd, X86_64_CC_E, 2 + 3);                         /* 6 bytes */
1393
1394                         x86_64_cqto(cd);
1395                         x86_64_idiv_reg(cd, REG_ITMP3);
1396
1397                         if (iptr->dst->flags & INMEMORY) {
1398                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1399                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1400
1401                         } else {
1402                                 M_INTMOVE(RDX, iptr->dst->regoff);
1403
1404                                 if (iptr->dst->regoff != RDX) {
1405                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1406                                 }
1407                         }
1408                         break;
1409
1410                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1411                                       /* val.i = constant                             */
1412
1413                         var_to_reg_int(s1, src, REG_ITMP1);
1414                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1415                         M_INTMOVE(s1, REG_ITMP1);
1416                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1417                         x86_64_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1418                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1419                         x86_64_shift_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1420                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1421                         store_reg_to_var_int(iptr->dst, d);
1422                         break;
1423
1424                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1425                                       /* val.l = constant                             */
1426
1427                         var_to_reg_int(s1, src, REG_ITMP1);
1428                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1429                         M_INTMOVE(s1, REG_ITMP1);
1430                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1431                         x86_64_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1432                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1433                         x86_64_alu_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1434                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1435                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1436                         store_reg_to_var_int(iptr->dst, d);
1437                         break;
1438
1439                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1440
1441                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1442                         x86_64_emit_ishift(cd, X86_64_SHL, src, iptr);
1443                         break;
1444
1445                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1446                                       /* val.i = constant                             */
1447
1448                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1449                         x86_64_emit_ishiftconst(cd, X86_64_SHL, src, iptr);
1450                         break;
1451
1452                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1453
1454                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1455                         x86_64_emit_ishift(cd, X86_64_SAR, src, iptr);
1456                         break;
1457
1458                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1459                                       /* val.i = constant                             */
1460
1461                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1462                         x86_64_emit_ishiftconst(cd, X86_64_SAR, src, iptr);
1463                         break;
1464
1465                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1466
1467                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1468                         x86_64_emit_ishift(cd, X86_64_SHR, src, iptr);
1469                         break;
1470
1471                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1472                                       /* val.i = constant                             */
1473
1474                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1475                         x86_64_emit_ishiftconst(cd, X86_64_SHR, src, iptr);
1476                         break;
1477
1478                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1479
1480                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1481                         x86_64_emit_lshift(cd, X86_64_SHL, src, iptr);
1482                         break;
1483
1484         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1485                                           /* val.i = constant                             */
1486
1487                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1488                         x86_64_emit_lshiftconst(cd, X86_64_SHL, src, iptr);
1489                         break;
1490
1491                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1492
1493                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1494                         x86_64_emit_lshift(cd, X86_64_SAR, src, iptr);
1495                         break;
1496
1497                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1498                                       /* val.i = constant                             */
1499
1500                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1501                         x86_64_emit_lshiftconst(cd, X86_64_SAR, src, iptr);
1502                         break;
1503
1504                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1505
1506                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1507                         x86_64_emit_lshift(cd, X86_64_SHR, src, iptr);
1508                         break;
1509
1510                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1511                                       /* val.l = constant                             */
1512
1513                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1514                         x86_64_emit_lshiftconst(cd, X86_64_SHR, src, iptr);
1515                         break;
1516
1517                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1518
1519                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1520                         x86_64_emit_ialu(cd, X86_64_AND, src, iptr);
1521                         break;
1522
1523                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1524                                       /* val.i = constant                             */
1525
1526                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1527                         x86_64_emit_ialuconst(cd, X86_64_AND, src, iptr);
1528                         break;
1529
1530                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1531
1532                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1533                         x86_64_emit_lalu(cd, X86_64_AND, src, iptr);
1534                         break;
1535
1536                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1537                                       /* val.l = constant                             */
1538
1539                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1540                         x86_64_emit_laluconst(cd, X86_64_AND, src, iptr);
1541                         break;
1542
1543                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1544
1545                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1546                         x86_64_emit_ialu(cd, X86_64_OR, src, iptr);
1547                         break;
1548
1549                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1550                                       /* val.i = constant                             */
1551
1552                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1553                         x86_64_emit_ialuconst(cd, X86_64_OR, src, iptr);
1554                         break;
1555
1556                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1557
1558                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1559                         x86_64_emit_lalu(cd, X86_64_OR, src, iptr);
1560                         break;
1561
1562                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1563                                       /* val.l = constant                             */
1564
1565                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1566                         x86_64_emit_laluconst(cd, X86_64_OR, src, iptr);
1567                         break;
1568
1569                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1570
1571                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1572                         x86_64_emit_ialu(cd, X86_64_XOR, src, iptr);
1573                         break;
1574
1575                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1576                                       /* val.i = constant                             */
1577
1578                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1579                         x86_64_emit_ialuconst(cd, X86_64_XOR, src, iptr);
1580                         break;
1581
1582                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1583
1584                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1585                         x86_64_emit_lalu(cd, X86_64_XOR, src, iptr);
1586                         break;
1587
1588                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1589                                       /* val.l = constant                             */
1590
1591                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1592                         x86_64_emit_laluconst(cd, X86_64_XOR, src, iptr);
1593                         break;
1594
1595
1596                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1597                                       /* op1 = variable, val.i = constant             */
1598
1599                         /* using inc and dec is definitely faster than add -- tested      */
1600                         /* with sieve                                                     */
1601
1602                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1603                         d = var->regoff;
1604                         if (var->flags & INMEMORY) {
1605                                 if (iptr->val.i == 1) {
1606                                         x86_64_incl_membase(cd, REG_SP, d * 8);
1607
1608                                 } else if (iptr->val.i == -1) {
1609                                         x86_64_decl_membase(cd, REG_SP, d * 8);
1610
1611                                 } else {
1612                                         x86_64_alul_imm_membase(cd, X86_64_ADD, iptr->val.i, REG_SP, d * 8);
1613                                 }
1614
1615                         } else {
1616                                 if (iptr->val.i == 1) {
1617                                         x86_64_incl_reg(cd, d);
1618
1619                                 } else if (iptr->val.i == -1) {
1620                                         x86_64_decl_reg(cd, d);
1621
1622                                 } else {
1623                                         x86_64_alul_imm_reg(cd, X86_64_ADD, iptr->val.i, d);
1624                                 }
1625                         }
1626                         break;
1627
1628
1629                 /* floating operations ************************************************/
1630
1631                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1632
1633                         var_to_reg_flt(s1, src, REG_FTMP1);
1634                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1635                         a = dseg_adds4(cd, 0x80000000);
1636                         M_FLTMOVE(s1, d);
1637                         x86_64_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, REG_FTMP2);
1638                         x86_64_xorps_reg_reg(cd, REG_FTMP2, d);
1639                         store_reg_to_var_flt(iptr->dst, d);
1640                         break;
1641
1642                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1643
1644                         var_to_reg_flt(s1, src, REG_FTMP1);
1645                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1646                         a = dseg_adds8(cd, 0x8000000000000000);
1647                         M_FLTMOVE(s1, d);
1648                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, REG_FTMP2);
1649                         x86_64_xorpd_reg_reg(cd, REG_FTMP2, d);
1650                         store_reg_to_var_flt(iptr->dst, d);
1651                         break;
1652
1653                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1654
1655                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1656                         var_to_reg_flt(s2, src, REG_FTMP2);
1657                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1658                         if (s1 == d) {
1659                                 x86_64_addss_reg_reg(cd, s2, d);
1660                         } else if (s2 == d) {
1661                                 x86_64_addss_reg_reg(cd, s1, d);
1662                         } else {
1663                                 M_FLTMOVE(s1, d);
1664                                 x86_64_addss_reg_reg(cd, s2, d);
1665                         }
1666                         store_reg_to_var_flt(iptr->dst, d);
1667                         break;
1668
1669                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1670
1671                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1672                         var_to_reg_flt(s2, src, REG_FTMP2);
1673                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1674                         if (s1 == d) {
1675                                 x86_64_addsd_reg_reg(cd, s2, d);
1676                         } else if (s2 == d) {
1677                                 x86_64_addsd_reg_reg(cd, s1, d);
1678                         } else {
1679                                 M_FLTMOVE(s1, d);
1680                                 x86_64_addsd_reg_reg(cd, s2, d);
1681                         }
1682                         store_reg_to_var_flt(iptr->dst, d);
1683                         break;
1684
1685                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1686
1687                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1688                         var_to_reg_flt(s2, src, REG_FTMP2);
1689                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1690                         if (s2 == d) {
1691                                 M_FLTMOVE(s2, REG_FTMP2);
1692                                 s2 = REG_FTMP2;
1693                         }
1694                         M_FLTMOVE(s1, d);
1695                         x86_64_subss_reg_reg(cd, s2, d);
1696                         store_reg_to_var_flt(iptr->dst, d);
1697                         break;
1698
1699                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1700
1701                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1702                         var_to_reg_flt(s2, src, REG_FTMP2);
1703                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1704                         if (s2 == d) {
1705                                 M_FLTMOVE(s2, REG_FTMP2);
1706                                 s2 = REG_FTMP2;
1707                         }
1708                         M_FLTMOVE(s1, d);
1709                         x86_64_subsd_reg_reg(cd, s2, d);
1710                         store_reg_to_var_flt(iptr->dst, d);
1711                         break;
1712
1713                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1714
1715                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1716                         var_to_reg_flt(s2, src, REG_FTMP2);
1717                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1718                         if (s1 == d) {
1719                                 x86_64_mulss_reg_reg(cd, s2, d);
1720                         } else if (s2 == d) {
1721                                 x86_64_mulss_reg_reg(cd, s1, d);
1722                         } else {
1723                                 M_FLTMOVE(s1, d);
1724                                 x86_64_mulss_reg_reg(cd, s2, d);
1725                         }
1726                         store_reg_to_var_flt(iptr->dst, d);
1727                         break;
1728
1729                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1730
1731                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1732                         var_to_reg_flt(s2, src, REG_FTMP2);
1733                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1734                         if (s1 == d) {
1735                                 x86_64_mulsd_reg_reg(cd, s2, d);
1736                         } else if (s2 == d) {
1737                                 x86_64_mulsd_reg_reg(cd, s1, d);
1738                         } else {
1739                                 M_FLTMOVE(s1, d);
1740                                 x86_64_mulsd_reg_reg(cd, s2, d);
1741                         }
1742                         store_reg_to_var_flt(iptr->dst, d);
1743                         break;
1744
1745                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1746
1747                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1748                         var_to_reg_flt(s2, src, REG_FTMP2);
1749                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1750                         if (s2 == d) {
1751                                 M_FLTMOVE(s2, REG_FTMP2);
1752                                 s2 = REG_FTMP2;
1753                         }
1754                         M_FLTMOVE(s1, d);
1755                         x86_64_divss_reg_reg(cd, s2, d);
1756                         store_reg_to_var_flt(iptr->dst, d);
1757                         break;
1758
1759                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1760
1761                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1762                         var_to_reg_flt(s2, src, REG_FTMP2);
1763                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1764                         if (s2 == d) {
1765                                 M_FLTMOVE(s2, REG_FTMP2);
1766                                 s2 = REG_FTMP2;
1767                         }
1768                         M_FLTMOVE(s1, d);
1769                         x86_64_divsd_reg_reg(cd, s2, d);
1770                         store_reg_to_var_flt(iptr->dst, d);
1771                         break;
1772
1773                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1774
1775                         var_to_reg_int(s1, src, REG_ITMP1);
1776                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1777                         x86_64_cvtsi2ss_reg_reg(cd, s1, d);
1778                         store_reg_to_var_flt(iptr->dst, d);
1779                         break;
1780
1781                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1782
1783                         var_to_reg_int(s1, src, REG_ITMP1);
1784                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1785                         x86_64_cvtsi2sd_reg_reg(cd, s1, d);
1786                         store_reg_to_var_flt(iptr->dst, d);
1787                         break;
1788
1789                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1790
1791                         var_to_reg_int(s1, src, REG_ITMP1);
1792                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1793                         x86_64_cvtsi2ssq_reg_reg(cd, s1, d);
1794                         store_reg_to_var_flt(iptr->dst, d);
1795                         break;
1796
1797                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1798
1799                         var_to_reg_int(s1, src, REG_ITMP1);
1800                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1801                         x86_64_cvtsi2sdq_reg_reg(cd, s1, d);
1802                         store_reg_to_var_flt(iptr->dst, d);
1803                         break;
1804
1805                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1806
1807                         var_to_reg_flt(s1, src, REG_FTMP1);
1808                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1809                         x86_64_cvttss2si_reg_reg(cd, s1, d);
1810                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1811                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1812                         x86_64_jcc(cd, X86_64_CC_NE, a);
1813                         M_FLTMOVE(s1, REG_FTMP1);
1814                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2i, REG_ITMP2);
1815                         x86_64_call_reg(cd, REG_ITMP2);
1816                         M_INTMOVE(REG_RESULT, d);
1817                         store_reg_to_var_int(iptr->dst, d);
1818                         break;
1819
1820                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1821
1822                         var_to_reg_flt(s1, src, REG_FTMP1);
1823                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1824                         x86_64_cvttsd2si_reg_reg(cd, s1, d);
1825                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1826                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1827                         x86_64_jcc(cd, X86_64_CC_NE, a);
1828                         M_FLTMOVE(s1, REG_FTMP1);
1829                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2i, REG_ITMP2);
1830                         x86_64_call_reg(cd, REG_ITMP2);
1831                         M_INTMOVE(REG_RESULT, d);
1832                         store_reg_to_var_int(iptr->dst, d);
1833                         break;
1834
1835                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1836
1837                         var_to_reg_flt(s1, src, REG_FTMP1);
1838                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1839                         x86_64_cvttss2siq_reg_reg(cd, s1, d);
1840                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1841                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1842                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1843                         x86_64_jcc(cd, X86_64_CC_NE, a);
1844                         M_FLTMOVE(s1, REG_FTMP1);
1845                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2l, REG_ITMP2);
1846                         x86_64_call_reg(cd, REG_ITMP2);
1847                         M_INTMOVE(REG_RESULT, d);
1848                         store_reg_to_var_int(iptr->dst, d);
1849                         break;
1850
1851                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1852
1853                         var_to_reg_flt(s1, src, REG_FTMP1);
1854                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1855                         x86_64_cvttsd2siq_reg_reg(cd, s1, d);
1856                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1857                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1858                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1859                         x86_64_jcc(cd, X86_64_CC_NE, a);
1860                         M_FLTMOVE(s1, REG_FTMP1);
1861                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2l, REG_ITMP2);
1862                         x86_64_call_reg(cd, REG_ITMP2);
1863                         M_INTMOVE(REG_RESULT, d);
1864                         store_reg_to_var_int(iptr->dst, d);
1865                         break;
1866
1867                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1868
1869                         var_to_reg_flt(s1, src, REG_FTMP1);
1870                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1871                         x86_64_cvtss2sd_reg_reg(cd, s1, d);
1872                         store_reg_to_var_flt(iptr->dst, d);
1873                         break;
1874
1875                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1876
1877                         var_to_reg_flt(s1, src, REG_FTMP1);
1878                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1879                         x86_64_cvtsd2ss_reg_reg(cd, s1, d);
1880                         store_reg_to_var_flt(iptr->dst, d);
1881                         break;
1882
1883                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1884                                           /* == => 0, < => 1, > => -1 */
1885
1886                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1887                         var_to_reg_flt(s2, src, REG_FTMP2);
1888                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1889                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1890                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1891                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1892                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1893                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1894                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1895                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1896                         store_reg_to_var_int(iptr->dst, d);
1897                         break;
1898
1899                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1900                                           /* == => 0, < => 1, > => -1 */
1901
1902                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1903                         var_to_reg_flt(s2, src, REG_FTMP2);
1904                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1905                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1906                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1907                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1908                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1909                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1910                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1911                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1912                         store_reg_to_var_int(iptr->dst, d);
1913                         break;
1914
1915                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1916                                           /* == => 0, < => 1, > => -1 */
1917
1918                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1919                         var_to_reg_flt(s2, src, REG_FTMP2);
1920                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1921                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1922                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1923                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1924                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1925                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1926                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1927                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1928                         store_reg_to_var_int(iptr->dst, d);
1929                         break;
1930
1931                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1932                                           /* == => 0, < => 1, > => -1 */
1933
1934                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1935                         var_to_reg_flt(s2, src, REG_FTMP2);
1936                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1937                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1938                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1939                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1940                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1941                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1942                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1943                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1944                         store_reg_to_var_int(iptr->dst, d);
1945                         break;
1946
1947
1948                 /* memory operations **************************************************/
1949
1950                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1951
1952                         var_to_reg_int(s1, src, REG_ITMP1);
1953                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1954                         gen_nullptr_check(s1);
1955                         x86_64_movl_membase_reg(cd, s1, OFFSET(java_arrayheader, size), d);
1956                         store_reg_to_var_int(iptr->dst, d);
1957                         break;
1958
1959                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1960
1961                         var_to_reg_int(s1, src->prev, REG_ITMP1);
1962                         var_to_reg_int(s2, src, REG_ITMP2);
1963                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1964                         if (iptr->op1 == 0) {
1965                                 gen_nullptr_check(s1);
1966                                 gen_bound_check;
1967                         }
1968                         x86_64_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1969                         store_reg_to_var_int(iptr->dst, d);
1970                         break;
1971
1972                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1973
1974                         var_to_reg_int(s1, src->prev, REG_ITMP1);
1975                         var_to_reg_int(s2, src, REG_ITMP2);
1976                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1977                         if (iptr->op1 == 0) {
1978                                 gen_nullptr_check(s1);
1979                                 gen_bound_check;
1980                         }
1981                         x86_64_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1982                         store_reg_to_var_int(iptr->dst, d);
1983                         break;
1984
1985                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1986
1987                         var_to_reg_int(s1, src->prev, REG_ITMP1);
1988                         var_to_reg_int(s2, src, REG_ITMP2);
1989                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1990                         if (iptr->op1 == 0) {
1991                                 gen_nullptr_check(s1);
1992                                 gen_bound_check;
1993                         }
1994                         x86_64_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1995                         store_reg_to_var_int(iptr->dst, d);
1996                         break;
1997
1998                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1999
2000                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2001                         var_to_reg_int(s2, src, REG_ITMP2);
2002                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2003                         if (iptr->op1 == 0) {
2004                                 gen_nullptr_check(s1);
2005                                 gen_bound_check;
2006                         }
2007                         x86_64_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2008                         store_reg_to_var_flt(iptr->dst, d);
2009                         break;
2010
2011                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2012
2013                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2014                         var_to_reg_int(s2, src, REG_ITMP2);
2015                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2016                         if (iptr->op1 == 0) {
2017                                 gen_nullptr_check(s1);
2018                                 gen_bound_check;
2019                         }
2020                         x86_64_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2021                         store_reg_to_var_flt(iptr->dst, d);
2022                         break;
2023
2024                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2025
2026                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2027                         var_to_reg_int(s2, src, REG_ITMP2);
2028                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2029                         if (iptr->op1 == 0) {
2030                                 gen_nullptr_check(s1);
2031                                 gen_bound_check;
2032                         }
2033                         x86_64_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2034                         store_reg_to_var_int(iptr->dst, d);
2035                         break;
2036
2037                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2038
2039                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2040                         var_to_reg_int(s2, src, REG_ITMP2);
2041                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2042                         if (iptr->op1 == 0) {
2043                                 gen_nullptr_check(s1);
2044                                 gen_bound_check;
2045                         }
2046                         x86_64_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2047                         store_reg_to_var_int(iptr->dst, d);
2048                         break;
2049
2050                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2051
2052                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2053                         var_to_reg_int(s2, src, REG_ITMP2);
2054                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2055                         if (iptr->op1 == 0) {
2056                                 gen_nullptr_check(s1);
2057                                 gen_bound_check;
2058                         }
2059                         x86_64_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2060                         store_reg_to_var_int(iptr->dst, d);
2061                         break;
2062
2063
2064                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2065
2066                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2067                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2068                         if (iptr->op1 == 0) {
2069                                 gen_nullptr_check(s1);
2070                                 gen_bound_check;
2071                         }
2072                         var_to_reg_int(s3, src, REG_ITMP3);
2073                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2074                         break;
2075
2076                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2077
2078                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2079                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2080                         if (iptr->op1 == 0) {
2081                                 gen_nullptr_check(s1);
2082                                 gen_bound_check;
2083                         }
2084                         var_to_reg_int(s3, src, REG_ITMP3);
2085                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2086                         break;
2087
2088                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2089
2090                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2091                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2092                         if (iptr->op1 == 0) {
2093                                 gen_nullptr_check(s1);
2094                                 gen_bound_check;
2095                         }
2096                         var_to_reg_int(s3, src, REG_ITMP3);
2097                         x86_64_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2098                         break;
2099
2100                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2101
2102                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2103                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2104                         if (iptr->op1 == 0) {
2105                                 gen_nullptr_check(s1);
2106                                 gen_bound_check;
2107                         }
2108                         var_to_reg_flt(s3, src, REG_FTMP3);
2109                         x86_64_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2110                         break;
2111
2112                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2113
2114                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2115                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2116                         if (iptr->op1 == 0) {
2117                                 gen_nullptr_check(s1);
2118                                 gen_bound_check;
2119                         }
2120                         var_to_reg_flt(s3, src, REG_FTMP3);
2121                         x86_64_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2122                         break;
2123
2124                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2125
2126                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2127                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2128                         if (iptr->op1 == 0) {
2129                                 gen_nullptr_check(s1);
2130                                 gen_bound_check;
2131                         }
2132                         var_to_reg_int(s3, src, REG_ITMP3);
2133                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2134                         break;
2135
2136                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2137
2138                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2139                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2140                         if (iptr->op1 == 0) {
2141                                 gen_nullptr_check(s1);
2142                                 gen_bound_check;
2143                         }
2144                         var_to_reg_int(s3, src, REG_ITMP3);
2145                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2146                         break;
2147
2148                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2149
2150                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2151                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2152                         if (iptr->op1 == 0) {
2153                                 gen_nullptr_check(s1);
2154                                 gen_bound_check;
2155                         }
2156                         var_to_reg_int(s3, src, REG_ITMP3);
2157                         x86_64_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2158                         break;
2159
2160                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2161
2162                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2163                         var_to_reg_int(s2, src, REG_ITMP2);
2164                         if (iptr->op1 == 0) {
2165                                 gen_nullptr_check(s1);
2166                                 gen_bound_check;
2167                         }
2168                         x86_64_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2169                         break;
2170
2171                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2172
2173                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2174                         var_to_reg_int(s2, src, REG_ITMP2);
2175                         if (iptr->op1 == 0) {
2176                                 gen_nullptr_check(s1);
2177                                 gen_bound_check;
2178                         }
2179
2180                         if (IS_IMM32(iptr->val.l)) {
2181                                 x86_64_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2182
2183                         } else {
2184                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2185                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2186                         }
2187                         break;
2188
2189                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2190
2191                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2192                         var_to_reg_int(s2, src, REG_ITMP2);
2193                         if (iptr->op1 == 0) {
2194                                 gen_nullptr_check(s1);
2195                                 gen_bound_check;
2196                         }
2197                         x86_64_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2198                         break;
2199
2200                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2201
2202                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2203                         var_to_reg_int(s2, src, REG_ITMP2);
2204                         if (iptr->op1 == 0) {
2205                                 gen_nullptr_check(s1);
2206                                 gen_bound_check;
2207                         }
2208                         x86_64_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2209                         break;
2210
2211                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2212
2213                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2214                         var_to_reg_int(s2, src, REG_ITMP2);
2215                         if (iptr->op1 == 0) {
2216                                 gen_nullptr_check(s1);
2217                                 gen_bound_check;
2218                         }
2219                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2220                         break;
2221
2222                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2223
2224                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2225                         var_to_reg_int(s2, src, REG_ITMP2);
2226                         if (iptr->op1 == 0) {
2227                                 gen_nullptr_check(s1);
2228                                 gen_bound_check;
2229                         }
2230                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2231                         break;
2232
2233
2234                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2235                                       /* op1 = type, val.a = field address            */
2236
2237                         if (!iptr->val.a) {
2238                                 codegen_addpatchref(cd, cd->mcodeptr,
2239                                                                         PATCHER_get_putstatic,
2240                                                                         (unresolved_field *) iptr->target);
2241
2242                                 if (showdisassemble) {
2243                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2244                                 }
2245
2246                                 a = 0;
2247
2248                         } else {
2249                                 fieldinfo *fi = iptr->val.a;
2250
2251                                 if (!fi->class->initialized) {
2252                                         codegen_addpatchref(cd, cd->mcodeptr,
2253                                                                                 PATCHER_clinit, fi->class);
2254
2255                                         if (showdisassemble) {
2256                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2257                                         }
2258                                 }
2259
2260                                 a = (ptrint) &(fi->value);
2261                         }
2262
2263                         /* This approach is much faster than moving the field address     */
2264                         /* inline into a register. */
2265                         a = dseg_addaddress(cd, a);
2266                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP2);
2267                         switch (iptr->op1) {
2268                         case TYPE_INT:
2269                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2270                                 x86_64_movl_membase_reg(cd, REG_ITMP2, 0, d);
2271                                 store_reg_to_var_int(iptr->dst, d);
2272                                 break;
2273                         case TYPE_LNG:
2274                         case TYPE_ADR:
2275                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2276                                 x86_64_mov_membase_reg(cd, REG_ITMP2, 0, d);
2277                                 store_reg_to_var_int(iptr->dst, d);
2278                                 break;
2279                         case TYPE_FLT:
2280                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2281                                 x86_64_movss_membase_reg(cd, REG_ITMP2, 0, d);
2282                                 store_reg_to_var_flt(iptr->dst, d);
2283                                 break;
2284                         case TYPE_DBL:
2285                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2286                                 x86_64_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2287                                 store_reg_to_var_flt(iptr->dst, d);
2288                                 break;
2289                         }
2290                         break;
2291
2292                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2293                                       /* op1 = type, val.a = field address            */
2294
2295                         if (!iptr->val.a) {
2296                                 codegen_addpatchref(cd, cd->mcodeptr,
2297                                                                         PATCHER_get_putstatic,
2298                                                                         (unresolved_field *) iptr->target);
2299
2300                                 if (showdisassemble) {
2301                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2302                                 }
2303
2304                                 a = 0;
2305
2306                         } else {
2307                                 fieldinfo *fi = iptr->val.a;
2308
2309                                 if (!fi->class->initialized) {
2310                                         codegen_addpatchref(cd, cd->mcodeptr,
2311                                                                                 PATCHER_clinit, fi->class);
2312
2313                                         if (showdisassemble) {
2314                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2315                                         }
2316                                 }
2317
2318                                 a = (ptrint) &(fi->value);
2319                         }
2320
2321                         /* This approach is much faster than moving the field address     */
2322                         /* inline into a register. */
2323                         a = dseg_addaddress(cd, a);
2324                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP2);
2325                         switch (iptr->op1) {
2326                         case TYPE_INT:
2327                                 var_to_reg_int(s2, src, REG_ITMP1);
2328                                 x86_64_movl_reg_membase(cd, s2, REG_ITMP2, 0);
2329                                 break;
2330                         case TYPE_LNG:
2331                         case TYPE_ADR:
2332                                 var_to_reg_int(s2, src, REG_ITMP1);
2333                                 x86_64_mov_reg_membase(cd, s2, REG_ITMP2, 0);
2334                                 break;
2335                         case TYPE_FLT:
2336                                 var_to_reg_flt(s2, src, REG_FTMP1);
2337                                 x86_64_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2338                                 break;
2339                         case TYPE_DBL:
2340                                 var_to_reg_flt(s2, src, REG_FTMP1);
2341                                 x86_64_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2342                                 break;
2343                         }
2344                         break;
2345
2346                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2347                                           /* val = value (in current instruction)     */
2348                                           /* op1 = type, val.a = field address (in    */
2349                                           /* following NOP)                           */
2350
2351                         if (!iptr[1].val.a) {
2352                                 codegen_addpatchref(cd, cd->mcodeptr,
2353                                                                         PATCHER_get_putstatic,
2354                                                                         (unresolved_field *) iptr[1].target);
2355
2356                                 if (showdisassemble) {
2357                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2358                                 }
2359
2360                                 a = 0;
2361
2362                         } else {
2363                                 fieldinfo *fi = iptr[1].val.a;
2364
2365                                 if (!fi->class->initialized) {
2366                                         codegen_addpatchref(cd, cd->mcodeptr,
2367                                                                                 PATCHER_clinit, fi->class);
2368
2369                                         if (showdisassemble) {
2370                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2371                                         }
2372                                 }
2373
2374                                 a = (ptrint) &(fi->value);
2375                         }
2376
2377                         /* This approach is much faster than moving the field address     */
2378                         /* inline into a register. */
2379                         a = dseg_addaddress(cd, a);
2380                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP1);
2381                         switch (iptr->op1) {
2382                         case TYPE_INT:
2383                         case TYPE_FLT:
2384                                 x86_64_movl_imm_membase(cd, iptr->val.i, REG_ITMP1, 0);
2385                                 break;
2386                         case TYPE_LNG:
2387                         case TYPE_ADR:
2388                         case TYPE_DBL:
2389                                 if (IS_IMM32(iptr->val.l)) {
2390                                         x86_64_mov_imm_membase(cd, iptr->val.l, REG_ITMP1, 0);
2391                                 } else {
2392                                         x86_64_movl_imm_membase(cd, iptr->val.l, REG_ITMP1, 0);
2393                                         x86_64_movl_imm_membase(cd, iptr->val.l >> 32, REG_ITMP1, 4);
2394                                 }
2395                                 break;
2396                         }
2397                         break;
2398
2399                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2400                                       /* op1 = type, val.i = field offset             */
2401
2402                         var_to_reg_int(s1, src, REG_ITMP1);
2403                         gen_nullptr_check(s1);
2404
2405                         if (!iptr->val.a) {
2406                                 codegen_addpatchref(cd, cd->mcodeptr,
2407                                                                         PATCHER_get_putfield,
2408                                                                         (unresolved_field *) iptr->target);
2409
2410                                 if (showdisassemble) {
2411                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2412                                 }
2413
2414                                 a = 0;
2415
2416                         } else {
2417                                 a = ((fieldinfo *) (iptr->val.a))->offset;
2418                         }
2419
2420                         switch (iptr->op1) {
2421                         case TYPE_INT:
2422                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2423                                 x86_64_movl_membase32_reg(cd, s1, a, d);
2424                                 store_reg_to_var_int(iptr->dst, d);
2425                                 break;
2426                         case TYPE_LNG:
2427                         case TYPE_ADR:
2428                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2429                                 x86_64_mov_membase32_reg(cd, s1, a, d);
2430                                 store_reg_to_var_int(iptr->dst, d);
2431                                 break;
2432                         case TYPE_FLT:
2433                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2434                                 x86_64_movss_membase32_reg(cd, s1, a, d);
2435                                 store_reg_to_var_flt(iptr->dst, d);
2436                                 break;
2437                         case TYPE_DBL:
2438                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2439                                 x86_64_movsd_membase32_reg(cd, s1, a, d);
2440                                 store_reg_to_var_flt(iptr->dst, d);
2441                                 break;
2442                         }
2443                         break;
2444
2445                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2446                                       /* op1 = type, val.i = field offset             */
2447
2448                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2449                         gen_nullptr_check(s1);
2450                         if (IS_INT_LNG_TYPE(iptr->op1)) {
2451                                 var_to_reg_int(s2, src, REG_ITMP2);
2452                         } else {
2453                                 var_to_reg_flt(s2, src, REG_FTMP2);
2454                         }
2455
2456                         if (!iptr->val.a) {
2457                                 codegen_addpatchref(cd, cd->mcodeptr,
2458                                                                         PATCHER_get_putfield,
2459                                                                         (unresolved_field *) iptr->target);
2460
2461                                 if (showdisassemble) {
2462                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2463                                 }
2464
2465                                 a = 0;
2466
2467                         } else {
2468                                 a = ((fieldinfo *) (iptr->val.a))->offset;
2469                         }
2470
2471                         switch (iptr->op1) {
2472                         case TYPE_INT:
2473                                 x86_64_movl_reg_membase32(cd, s2, s1, a);
2474                                 break;
2475                         case TYPE_LNG:
2476                         case TYPE_ADR:
2477                                 x86_64_mov_reg_membase32(cd, s2, s1, a);
2478                                 break;
2479                         case TYPE_FLT:
2480                                 x86_64_movss_reg_membase32(cd, s2, s1, a);
2481                                 break;
2482                         case TYPE_DBL:
2483                                 x86_64_movsd_reg_membase32(cd, s2, s1, a);
2484                                 break;
2485                         }
2486                         break;
2487
2488                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2489                                           /* val = value (in current instruction)     */
2490                                           /* op1 = type, val.a = field address (in    */
2491                                           /* following NOP)                           */
2492
2493                         var_to_reg_int(s1, src, REG_ITMP1);
2494                         gen_nullptr_check(s1);
2495
2496                         if (!iptr[1].val.a) {
2497                                 codegen_addpatchref(cd, cd->mcodeptr,
2498                                                                         PATCHER_putfieldconst,
2499                                                                         (unresolved_field *) iptr[1].target);
2500
2501                                 if (showdisassemble) {
2502                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2503                                 }
2504
2505                                 a = 0;
2506
2507                         } else {
2508                                 a = ((fieldinfo *) (iptr[1].val.a))->offset;
2509                         }
2510
2511                         switch (iptr->op1) {
2512                         case TYPE_INT:
2513                         case TYPE_FLT:
2514                                 x86_64_movl_imm_membase32(cd, iptr->val.i, s1, a);
2515                                 break;
2516                         case TYPE_LNG:
2517                         case TYPE_ADR:
2518                         case TYPE_DBL:
2519                                 /* We can only optimize the move, if the class is resolved.   */
2520                                 /* Otherwise we don't know what to patch.                     */
2521                                 if (iptr[1].val.a && IS_IMM32(iptr->val.l)) {
2522                                         x86_64_mov_imm_membase32(cd, iptr->val.l, s1, a);
2523                                 } else {
2524                                         x86_64_movl_imm_membase32(cd, iptr->val.l, s1, a);
2525                                         x86_64_movl_imm_membase32(cd, iptr->val.l >> 32, s1, a + 4);
2526                                 }
2527                                 break;
2528                         }
2529                         break;
2530
2531
2532                 /* branch operations **************************************************/
2533
2534                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2535
2536                         var_to_reg_int(s1, src, REG_ITMP1);
2537                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2538
2539                         x86_64_call_imm(cd, 0); /* passing exception pointer              */
2540                         x86_64_pop_reg(cd, REG_ITMP2_XPC);
2541
2542                         x86_64_mov_imm_reg(cd, (ptrint) asm_handle_exception, REG_ITMP3);
2543                         x86_64_jmp_reg(cd, REG_ITMP3);
2544                         break;
2545
2546                 case ICMD_GOTO:         /* ... ==> ...                                */
2547                                         /* op1 = target JavaVM pc                     */
2548
2549                         x86_64_jmp_imm(cd, 0);
2550                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2551                         break;
2552
2553                 case ICMD_JSR:          /* ... ==> ...                                */
2554                                         /* op1 = target JavaVM pc                     */
2555
2556                         x86_64_call_imm(cd, 0);
2557                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2558                         break;
2559
2560                 case ICMD_RET:          /* ... ==> ...                                */
2561                                         /* op1 = local variable                       */
2562
2563                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2564                         var_to_reg_int(s1, var, REG_ITMP1);
2565                         x86_64_jmp_reg(cd, s1);
2566                         break;
2567
2568                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2569                                         /* op1 = target JavaVM pc                     */
2570
2571                         if (src->flags & INMEMORY) {
2572                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2573
2574                         } else {
2575                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2576                         }
2577                         x86_64_jcc(cd, X86_64_CC_E, 0);
2578                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2579                         break;
2580
2581                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2582                                         /* op1 = target JavaVM pc                     */
2583
2584                         if (src->flags & INMEMORY) {
2585                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2586
2587                         } else {
2588                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2589                         }
2590                         x86_64_jcc(cd, X86_64_CC_NE, 0);
2591                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2592                         break;
2593
2594                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2595                                         /* op1 = target JavaVM pc, val.i = constant   */
2596
2597                         x86_64_emit_ifcc(cd, X86_64_CC_E, src, iptr);
2598                         break;
2599
2600                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2601                                         /* op1 = target JavaVM pc, val.i = constant   */
2602
2603                         x86_64_emit_ifcc(cd, X86_64_CC_L, src, iptr);
2604                         break;
2605
2606                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2607                                         /* op1 = target JavaVM pc, val.i = constant   */
2608
2609                         x86_64_emit_ifcc(cd, X86_64_CC_LE, src, iptr);
2610                         break;
2611
2612                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2613                                         /* op1 = target JavaVM pc, val.i = constant   */
2614
2615                         x86_64_emit_ifcc(cd, X86_64_CC_NE, src, iptr);
2616                         break;
2617
2618                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2619                                         /* op1 = target JavaVM pc, val.i = constant   */
2620
2621                         x86_64_emit_ifcc(cd, X86_64_CC_G, src, iptr);
2622                         break;
2623
2624                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2625                                         /* op1 = target JavaVM pc, val.i = constant   */
2626
2627                         x86_64_emit_ifcc(cd, X86_64_CC_GE, src, iptr);
2628                         break;
2629
2630                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2631                                         /* op1 = target JavaVM pc, val.l = constant   */
2632
2633                         x86_64_emit_if_lcc(cd, X86_64_CC_E, src, iptr);
2634                         break;
2635
2636                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2637                                         /* op1 = target JavaVM pc, val.l = constant   */
2638
2639                         x86_64_emit_if_lcc(cd, X86_64_CC_L, src, iptr);
2640                         break;
2641
2642                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2643                                         /* op1 = target JavaVM pc, val.l = constant   */
2644
2645                         x86_64_emit_if_lcc(cd, X86_64_CC_LE, src, iptr);
2646                         break;
2647
2648                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2649                                         /* op1 = target JavaVM pc, val.l = constant   */
2650
2651                         x86_64_emit_if_lcc(cd, X86_64_CC_NE, src, iptr);
2652                         break;
2653
2654                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2655                                         /* op1 = target JavaVM pc, val.l = constant   */
2656
2657                         x86_64_emit_if_lcc(cd, X86_64_CC_G, src, iptr);
2658                         break;
2659
2660                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2661                                         /* op1 = target JavaVM pc, val.l = constant   */
2662
2663                         x86_64_emit_if_lcc(cd, X86_64_CC_GE, src, iptr);
2664                         break;
2665
2666                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2667                                         /* op1 = target JavaVM pc                     */
2668
2669                         x86_64_emit_if_icmpcc(cd, X86_64_CC_E, src, iptr);
2670                         break;
2671
2672                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2673                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2674
2675                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_E, src, iptr);
2676                         break;
2677
2678                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2679                                         /* op1 = target JavaVM pc                     */
2680
2681                         x86_64_emit_if_icmpcc(cd, X86_64_CC_NE, src, iptr);
2682                         break;
2683
2684                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2685                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2686
2687                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_NE, src, iptr);
2688                         break;
2689
2690                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2691                                         /* op1 = target JavaVM pc                     */
2692
2693                         x86_64_emit_if_icmpcc(cd, X86_64_CC_L, src, iptr);
2694                         break;
2695
2696                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2697                                     /* op1 = target JavaVM pc                     */
2698
2699                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_L, src, iptr);
2700                         break;
2701
2702                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2703                                         /* op1 = target JavaVM pc                     */
2704
2705                         x86_64_emit_if_icmpcc(cd, X86_64_CC_G, src, iptr);
2706                         break;
2707
2708                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2709                                 /* op1 = target JavaVM pc                     */
2710
2711                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_G, src, iptr);
2712                         break;
2713
2714                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2715                                         /* op1 = target JavaVM pc                     */
2716
2717                         x86_64_emit_if_icmpcc(cd, X86_64_CC_LE, src, iptr);
2718                         break;
2719
2720                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2721                                         /* op1 = target JavaVM pc                     */
2722
2723                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_LE, src, iptr);
2724                         break;
2725
2726                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2727                                         /* op1 = target JavaVM pc                     */
2728
2729                         x86_64_emit_if_icmpcc(cd, X86_64_CC_GE, src, iptr);
2730                         break;
2731
2732                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2733                                     /* op1 = target JavaVM pc                     */
2734
2735                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_GE, src, iptr);
2736                         break;
2737
2738                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2739
2740                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2741                         break;
2742
2743                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2744                                         /* val.i = constant                           */
2745
2746                         var_to_reg_int(s1, src, REG_ITMP1);
2747                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2748                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2749                                 if (s1 == d) {
2750                                         M_INTMOVE(s1, REG_ITMP1);
2751                                         s1 = REG_ITMP1;
2752                                 }
2753                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2754                         }
2755                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2756                         x86_64_testl_reg_reg(cd, s1, s1);
2757                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_E, REG_ITMP2, d);
2758                         store_reg_to_var_int(iptr->dst, d);
2759                         break;
2760
2761                 case ICMD_IFNE_ICONST:  /* ..., value ==> ..., constant               */
2762                                         /* val.i = constant                           */
2763
2764                         var_to_reg_int(s1, src, REG_ITMP1);
2765                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2766                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2767                                 if (s1 == d) {
2768                                         M_INTMOVE(s1, REG_ITMP1);
2769                                         s1 = REG_ITMP1;
2770                                 }
2771                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2772                         }
2773                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2774                         x86_64_testl_reg_reg(cd, s1, s1);
2775                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_NE, REG_ITMP2, d);
2776                         store_reg_to_var_int(iptr->dst, d);
2777                         break;
2778
2779                 case ICMD_IFLT_ICONST:  /* ..., value ==> ..., constant               */
2780                                         /* val.i = constant                           */
2781
2782                         var_to_reg_int(s1, src, REG_ITMP1);
2783                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2784                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2785                                 if (s1 == d) {
2786                                         M_INTMOVE(s1, REG_ITMP1);
2787                                         s1 = REG_ITMP1;
2788                                 }
2789                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2790                         }
2791                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2792                         x86_64_testl_reg_reg(cd, s1, s1);
2793                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_L, REG_ITMP2, d);
2794                         store_reg_to_var_int(iptr->dst, d);
2795                         break;
2796
2797                 case ICMD_IFGE_ICONST:  /* ..., value ==> ..., constant               */
2798                                         /* val.i = constant                           */
2799
2800                         var_to_reg_int(s1, src, REG_ITMP1);
2801                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2802                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2803                                 if (s1 == d) {
2804                                         M_INTMOVE(s1, REG_ITMP1);
2805                                         s1 = REG_ITMP1;
2806                                 }
2807                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2808                         }
2809                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2810                         x86_64_testl_reg_reg(cd, s1, s1);
2811                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_GE, REG_ITMP2, d);
2812                         store_reg_to_var_int(iptr->dst, d);
2813                         break;
2814
2815                 case ICMD_IFGT_ICONST:  /* ..., value ==> ..., constant               */
2816                                         /* val.i = constant                           */
2817
2818                         var_to_reg_int(s1, src, REG_ITMP1);
2819                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2820                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2821                                 if (s1 == d) {
2822                                         M_INTMOVE(s1, REG_ITMP1);
2823                                         s1 = REG_ITMP1;
2824                                 }
2825                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2826                         }
2827                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2828                         x86_64_testl_reg_reg(cd, s1, s1);
2829                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP2, d);
2830                         store_reg_to_var_int(iptr->dst, d);
2831                         break;
2832
2833                 case ICMD_IFLE_ICONST:  /* ..., value ==> ..., constant               */
2834                                         /* val.i = constant                           */
2835
2836                         var_to_reg_int(s1, src, REG_ITMP1);
2837                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2838                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2839                                 if (s1 == d) {
2840                                         M_INTMOVE(s1, REG_ITMP1);
2841                                         s1 = REG_ITMP1;
2842                                 }
2843                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2844                         }
2845                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2846                         x86_64_testl_reg_reg(cd, s1, s1);
2847                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, d);
2848                         store_reg_to_var_int(iptr->dst, d);
2849                         break;
2850
2851
2852                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2853                 case ICMD_LRETURN:
2854                 case ICMD_ARETURN:
2855
2856                         var_to_reg_int(s1, src, REG_RESULT);
2857                         M_INTMOVE(s1, REG_RESULT);
2858
2859                         goto nowperformreturn;
2860
2861                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2862                 case ICMD_DRETURN:
2863
2864                         var_to_reg_flt(s1, src, REG_FRESULT);
2865                         M_FLTMOVE(s1, REG_FRESULT);
2866
2867                         goto nowperformreturn;
2868
2869                 case ICMD_RETURN:      /* ...  ==> ...                                */
2870
2871 nowperformreturn:
2872                         {
2873                         s4 i, p;
2874
2875                         p = parentargs_base;
2876
2877                         /* call trace function */
2878                         if (runverbose) {
2879                                 x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
2880
2881                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2882                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2883
2884                                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
2885                                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2886                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2887                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2888
2889                                 x86_64_mov_imm_reg(cd, (u8) builtin_displaymethodstop, REG_ITMP1);
2890                                 x86_64_call_reg(cd, REG_ITMP1);
2891
2892                                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2893                                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2894
2895                                 x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
2896                         }
2897
2898 #if defined(USE_THREADS)
2899                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2900                                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, rd->argintregs[0]);
2901
2902                                 /* we need to save the proper return value */
2903                                 switch (iptr->opc) {
2904                                 case ICMD_IRETURN:
2905                                 case ICMD_ARETURN:
2906                                 case ICMD_LRETURN:
2907                                         x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, rd->maxmemuse * 8);
2908                                         break;
2909                                 case ICMD_FRETURN:
2910                                 case ICMD_DRETURN:
2911                                         x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, rd->maxmemuse * 8);
2912                                         break;
2913                                 }
2914
2915                                 x86_64_mov_imm_reg(cd, (ptrint) builtin_monitorexit, REG_ITMP1);
2916                                 x86_64_call_reg(cd, REG_ITMP1);
2917
2918                                 /* and now restore the proper return value */
2919                                 switch (iptr->opc) {
2920                                 case ICMD_IRETURN:
2921                                 case ICMD_ARETURN:
2922                                 case ICMD_LRETURN:
2923                                         x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, REG_RESULT);
2924                                         break;
2925                                 case ICMD_FRETURN:
2926                                 case ICMD_DRETURN:
2927                                         x86_64_movq_membase_reg(cd, REG_SP, rd->maxmemuse * 8, REG_FRESULT);
2928                                         break;
2929                                 }
2930                         }
2931 #endif
2932
2933                         /* restore saved registers                                        */
2934                         for (i = rd->savintregcnt - 1; i >= rd->maxsavintreguse; i--) {
2935                                 p--; x86_64_mov_membase_reg(cd, REG_SP, p * 8, rd->savintregs[i]);
2936                         }
2937                         for (i = rd->savfltregcnt - 1; i >= rd->maxsavfltreguse; i--) {
2938                                 p--; x86_64_movq_membase_reg(cd, REG_SP, p * 8, rd->savfltregs[i]);
2939                         }
2940
2941                         /* deallocate stack                                               */
2942                         if (parentargs_base) {
2943                                 x86_64_alu_imm_reg(cd, X86_64_ADD, parentargs_base * 8, REG_SP);
2944                         }
2945
2946                         x86_64_ret(cd);
2947                         }
2948                         break;
2949
2950
2951                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2952                         {
2953                                 s4 i, l, *s4ptr;
2954                                 void **tptr;
2955
2956                                 tptr = (void **) iptr->target;
2957
2958                                 s4ptr = iptr->val.a;
2959                                 l = s4ptr[1];                          /* low     */
2960                                 i = s4ptr[2];                          /* high    */
2961
2962                                 var_to_reg_int(s1, src, REG_ITMP1);
2963                                 M_INTMOVE(s1, REG_ITMP1);
2964                                 if (l != 0) {
2965                                         x86_64_alul_imm_reg(cd, X86_64_SUB, l, REG_ITMP1);
2966                                 }
2967                                 i = i - l + 1;
2968
2969                 /* range check */
2970                                 x86_64_alul_imm_reg(cd, X86_64_CMP, i - 1, REG_ITMP1);
2971                                 x86_64_jcc(cd, X86_64_CC_A, 0);
2972
2973                 /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[0]), cd->mcodeptr); */
2974                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
2975
2976                                 /* build jump table top down and use address of lowest entry */
2977
2978                 /* s4ptr += 3 + i; */
2979                                 tptr += i;
2980
2981                                 while (--i >= 0) {
2982                                         /* dseg_addtarget(cd, BlockPtrOfPC(*--s4ptr)); */
2983                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
2984                                         --tptr;
2985                                 }
2986
2987                                 /* length of dataseg after last dseg_addtarget is used by load */
2988
2989                                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2);
2990                                 dseg_adddata(cd, cd->mcodeptr);
2991                                 x86_64_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2992                                 x86_64_jmp_reg(cd, REG_ITMP1);
2993                         }
2994                         break;
2995
2996
2997                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2998                         {
2999                                 s4 i, l, val, *s4ptr;
3000                                 void **tptr;
3001
3002                                 tptr = (void **) iptr->target;
3003
3004                                 s4ptr = iptr->val.a;
3005                                 l = s4ptr[0];                          /* default  */
3006                                 i = s4ptr[1];                          /* count    */
3007
3008                                 MCODECHECK((i<<2)+8);
3009                                 var_to_reg_int(s1, src, REG_ITMP1);    /* reg compare should always be faster */
3010                                 while (--i >= 0) {
3011                                         s4ptr += 2;
3012                                         ++tptr;
3013
3014                                         val = s4ptr[0];
3015                                         x86_64_alul_imm_reg(cd, X86_64_CMP, val, s1);
3016                                         x86_64_jcc(cd, X86_64_CC_E, 0);
3017                                         /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[1]), cd->mcodeptr); */
3018                                         codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3019                                 }
3020
3021                                 x86_64_jmp_imm(cd, 0);
3022                                 /* codegen_addreference(cd, BlockPtrOfPC(l), cd->mcodeptr); */
3023
3024                                 tptr = (void **) iptr->target;
3025                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3026                         }
3027                         break;
3028
3029
3030                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
3031                                         /* op1 = arg count val.a = builtintable entry */
3032
3033                         bte = iptr->val.a;
3034                         md = bte->md;
3035                         goto gen_method;
3036
3037                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
3038                                         /* op1 = arg count, val.a = method pointer    */
3039
3040                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
3041                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
3042                 case ICMD_INVOKEINTERFACE:
3043
3044                         lm = iptr->val.a;
3045
3046                         if (lm)
3047                                 md = lm->parseddesc;
3048                         else {
3049                                 unresolved_method *um = iptr->target;
3050                                 md = um->methodref->parseddesc.md;
3051                         }
3052
3053 gen_method:
3054                         s3 = iptr->op1;
3055
3056                         MCODECHECK((s3 << 1) + 64);
3057
3058                         /* copy arguments to registers or stack location */
3059
3060                         for (s3 = s3 - 1; s3 >= 0; s3--, src = src->prev) {
3061                                 if (src->varkind == ARGVAR)
3062                                         continue;
3063                                 if (IS_INT_LNG_TYPE(src->type)) {
3064                                         if (!md->params[s3].inmemory) {
3065                                                 s1 = rd->argintregs[md->params[s3].regoff];
3066                                                 var_to_reg_int(d, src, s1);
3067                                                 M_INTMOVE(d, s1);
3068                                         } else {
3069                                                 var_to_reg_int(d, src, REG_ITMP1);
3070                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
3071                                         }
3072
3073                                 } else {
3074                                         if (!md->params[s3].inmemory) {
3075                                                 s1 = rd->argfltregs[md->params[s3].regoff];
3076                                                 var_to_reg_flt(d, src, s1);
3077                                                 M_FLTMOVE(d, s1);
3078                                         } else {
3079                                                 var_to_reg_flt(d, src, REG_FTMP1);
3080                                                 M_DST(d, REG_SP, md->params[s3].regoff * 8);
3081                                         }
3082                                 }
3083                         } /* end of for */
3084
3085                         switch (iptr->opc) {
3086                         case ICMD_BUILTIN:
3087                                 if (iptr->target) {
3088                                         codegen_addpatchref(cd, cd->mcodeptr,
3089                                                                                 bte->fp, iptr->target);
3090
3091                                         if (showdisassemble) {
3092                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3093                                         }
3094
3095                                         a = 0;
3096
3097                                 } else {
3098                                         a = (ptrint) bte->fp;
3099                                 }
3100
3101                                 d = md->returntype.type;
3102
3103                                 x86_64_mov_imm_reg(cd, a, REG_ITMP1);
3104                                 x86_64_call_reg(cd, REG_ITMP1);
3105                                 break;
3106
3107                         case ICMD_INVOKESPECIAL:
3108                                 x86_64_test_reg_reg(cd, rd->argintregs[0], rd->argintregs[0]);
3109                                 x86_64_jcc(cd, X86_64_CC_Z, 0);
3110                                 codegen_addxnullrefs(cd, cd->mcodeptr);
3111
3112                                 /* first argument contains pointer */
3113 /*                              gen_nullptr_check(rd->argintregs[0]); */
3114
3115                                 /* access memory for hardware nullptr */
3116 /*                              x86_64_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); */
3117
3118                                 /* fall through */
3119
3120                         case ICMD_INVOKESTATIC:
3121                                 if (!lm) {
3122                                         unresolved_method *um = iptr->target;
3123
3124                                         codegen_addpatchref(cd, cd->mcodeptr,
3125                                                                                 PATCHER_invokestatic_special, um);
3126
3127                                         if (showdisassemble) {
3128                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3129                                         }
3130
3131                                         a = 0;
3132                                         d = um->methodref->parseddesc.md->returntype.type;
3133
3134                                 } else {
3135                                         a = (ptrint) lm->stubroutine;
3136                                         d = lm->parseddesc->returntype.type;
3137                                 }
3138
3139                                 x86_64_mov_imm_reg(cd, a, REG_ITMP2);
3140                                 x86_64_call_reg(cd, REG_ITMP2);
3141                                 break;
3142
3143                         case ICMD_INVOKEVIRTUAL:
3144                                 gen_nullptr_check(rd->argintregs[0]);
3145
3146                                 if (!lm) {
3147                                         unresolved_method *um = iptr->target;
3148
3149                                         codegen_addpatchref(cd, cd->mcodeptr,
3150                                                                                 PATCHER_invokevirtual, um);
3151
3152                                         if (showdisassemble) {
3153                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3154                                         }
3155
3156                                         s1 = 0;
3157                                         d = um->methodref->parseddesc.md->returntype.type;
3158
3159                                 } else {
3160                                         s1 = OFFSET(vftbl_t, table[0]) +
3161                                                 sizeof(methodptr) * lm->vftblindex;
3162                                         d = lm->parseddesc->returntype.type;
3163                                 }
3164
3165                                 x86_64_mov_membase_reg(cd, rd->argintregs[0],
3166                                                                            OFFSET(java_objectheader, vftbl),
3167                                                                            REG_ITMP2);
3168                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP1);
3169                                 x86_64_call_reg(cd, REG_ITMP1);
3170                                 break;
3171
3172                         case ICMD_INVOKEINTERFACE:
3173                                 gen_nullptr_check(rd->argintregs[0]);
3174
3175                                 if (!lm) {
3176                                         unresolved_method *um = iptr->target;
3177
3178                                         codegen_addpatchref(cd, cd->mcodeptr,
3179                                                                                 PATCHER_invokeinterface, um);
3180
3181                                         if (showdisassemble) {
3182                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3183                                         }
3184
3185                                         s1 = 0;
3186                                         s2 = 0;
3187                                         d = um->methodref->parseddesc.md->returntype.type;
3188
3189                                 } else {
3190                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3191                                                 sizeof(methodptr) * lm->class->index;
3192
3193                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3194
3195                                         d = lm->parseddesc->returntype.type;
3196                                 }
3197
3198                                 x86_64_mov_membase_reg(cd, rd->argintregs[0],
3199                                                                            OFFSET(java_objectheader, vftbl),
3200                                                                            REG_ITMP2);
3201                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP2);
3202                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s2, REG_ITMP1);
3203                                 x86_64_call_reg(cd, REG_ITMP1);
3204                                 break;
3205                         }
3206
3207                         /* d contains return type */
3208
3209                         if (d != TYPE_VOID) {
3210                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3211                                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3212                                         M_INTMOVE(REG_RESULT, s1);
3213                                         store_reg_to_var_int(iptr->dst, s1);
3214                                 } else {
3215                                         s1 = reg_of_var(rd, iptr->dst, REG_FRESULT);
3216                                         M_FLTMOVE(REG_FRESULT, s1);
3217                                         store_reg_to_var_flt(iptr->dst, s1);
3218                                 }
3219                         }
3220                         break;
3221
3222
3223                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3224
3225                                       /* op1:   0 == array, 1 == class                */
3226                                       /* val.a: (classinfo *) superclass              */
3227
3228                         /*  superclass is an interface:
3229                          *
3230                          *  OK if ((sub == NULL) ||
3231                          *         (sub->vftbl->interfacetablelength > super->index) &&
3232                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3233                          *
3234                          *  superclass is a class:
3235                          *
3236                          *  OK if ((sub == NULL) || (0
3237                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3238                          *         super->vftbl->diffval));
3239                          */
3240
3241                         {
3242                         classinfo *super;
3243                         vftbl_t   *supervftbl;
3244                         s4         superindex;
3245
3246                         super = (classinfo *) iptr->val.a;
3247
3248                         if (!super) {
3249                                 superindex = 0;
3250                                 supervftbl = NULL;
3251
3252                         } else {
3253                                 superindex = super->index;
3254                                 supervftbl = super->vftbl;
3255                         }
3256
3257 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3258             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3259 #endif
3260                         var_to_reg_int(s1, src, REG_ITMP1);
3261
3262                         /* calculate interface checkcast code size */
3263
3264                         s2 = 3; /* mov_membase_reg */
3265                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3266
3267                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3268                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3269                                 3 /* test */ + 6 /* jcc */;
3270
3271                         if (!super)
3272                                 s2 += (showdisassemble ? 5 : 0);
3273
3274                         /* calculate class checkcast code size */
3275
3276                         s3 = 3; /* mov_membase_reg */
3277                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3278                         s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3279
3280 #if 0
3281                         if (s1 != REG_ITMP1) {
3282                                 a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3283                                 CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3284                                 a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3285                                 CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3286                                 a += 3;    /* sub */
3287
3288                         } else
3289 #endif
3290                                 {
3291                                         s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3292                                                 10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3293                                         CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3294                                 }
3295
3296                         s3 += 3 /* cmp */ + 6 /* jcc */;
3297
3298                         if (!super)
3299                                 s3 += (showdisassemble ? 5 : 0);
3300
3301                         /* if class is not resolved, check which code to call */
3302
3303                         if (!super) {
3304                                 x86_64_test_reg_reg(cd, s1, s1);
3305                                 x86_64_jcc(cd, X86_64_CC_Z, 6 + (showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3306
3307                                 codegen_addpatchref(cd, cd->mcodeptr,
3308                                                                         PATCHER_checkcast_instanceof_flags,
3309                                                                         (constant_classref *) iptr->target);
3310
3311                                 if (showdisassemble) {
3312                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3313                                 }
3314
3315                                 x86_64_movl_imm_reg(cd, 0, REG_ITMP2); /* super->flags */
3316                                 x86_64_alul_imm_reg(cd, X86_64_AND, ACC_INTERFACE, REG_ITMP2);
3317                                 x86_64_jcc(cd, X86_64_CC_Z, s2 + 5);
3318                         }
3319
3320                         /* interface checkcast code */
3321
3322                         if (!super || (super->flags & ACC_INTERFACE)) {
3323                                 if (super) {
3324                                         x86_64_test_reg_reg(cd, s1, s1);
3325                                         x86_64_jcc(cd, X86_64_CC_Z, s2);
3326                                 }
3327
3328                                 x86_64_mov_membase_reg(cd, s1,
3329                                                                            OFFSET(java_objectheader, vftbl),
3330                                                                            REG_ITMP2);
3331
3332                                 if (!super) {
3333                                         codegen_addpatchref(cd, cd->mcodeptr,
3334                                                                                 PATCHER_checkcast_instanceof_interface,
3335                                                                                 (constant_classref *) iptr->target);
3336
3337                                         if (showdisassemble) {
3338                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3339                                         }
3340                                 }
3341
3342                                 x86_64_movl_membase32_reg(cd, REG_ITMP2,
3343                                                                                   OFFSET(vftbl_t, interfacetablelength),
3344                                                                                   REG_ITMP3);
3345                                 x86_64_alu_imm32_reg(cd, X86_64_SUB, superindex, REG_ITMP3);
3346                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3347                                 x86_64_jcc(cd, X86_64_CC_LE, 0);
3348                                 codegen_addxcastrefs(cd, cd->mcodeptr);
3349                                 x86_64_mov_membase32_reg(cd, REG_ITMP2,
3350                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3351                                                                                  superindex * sizeof(methodptr*),
3352                                                                                  REG_ITMP3);
3353                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3354                                 x86_64_jcc(cd, X86_64_CC_E, 0);
3355                                 codegen_addxcastrefs(cd, cd->mcodeptr);
3356
3357                                 if (!super)
3358                                         x86_64_jmp_imm(cd, s3);
3359                         }
3360
3361                         /* class checkcast code */
3362
3363                         if (!super || !(super->flags & ACC_INTERFACE)) {
3364                                 if (super) {
3365                                         x86_64_test_reg_reg(cd, s1, s1);
3366                                         x86_64_jcc(cd, X86_64_CC_Z, s3);
3367                                 }
3368
3369                                 x86_64_mov_membase_reg(cd, s1,
3370                                                                            OFFSET(java_objectheader, vftbl),
3371                                                                            REG_ITMP2);
3372
3373                                 if (!super) {
3374                                         codegen_addpatchref(cd, cd->mcodeptr,
3375                                                                                 PATCHER_checkcast_class,
3376                                                                                 (constant_classref *) iptr->target);
3377
3378                                         if (showdisassemble) {
3379                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3380                                         }
3381                                 }
3382
3383                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP3);
3384 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3385                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3386 #endif
3387                                 x86_64_movl_membase32_reg(cd, REG_ITMP2,
3388                                                                                   OFFSET(vftbl_t, baseval),
3389                                                                                   REG_ITMP2);
3390 /*                                      if (s1 != REG_ITMP1) { */
3391 /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3392 /*                                                                                              OFFSET(vftbl_t, baseval), */
3393 /*                                                                                              REG_ITMP1); */
3394 /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3395 /*                                                                                              OFFSET(vftbl_t, diffval), */
3396 /*                                                                                              REG_ITMP3); */
3397 /*  #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
3398 /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3399 /*  #endif */
3400 /*                                              x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP1, REG_ITMP2); */
3401
3402 /*                                      } else { */
3403                                 x86_64_movl_membase32_reg(cd, REG_ITMP3,
3404                                                                                   OFFSET(vftbl_t, baseval),
3405                                                                                   REG_ITMP3);
3406                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP3, REG_ITMP2);
3407                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP3);
3408                                 x86_64_movl_membase_reg(cd, REG_ITMP3,
3409                                                                                 OFFSET(vftbl_t, diffval),
3410                                                                                 REG_ITMP3);
3411 /*                                      } */
3412 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3413                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3414 #endif
3415                                 x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP3, REG_ITMP2);
3416                                 x86_64_jcc(cd, X86_64_CC_A, 0);    /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3417                                 codegen_addxcastrefs(cd, cd->mcodeptr);
3418                         }
3419                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3420                         M_INTMOVE(s1, d);
3421                         store_reg_to_var_int(iptr->dst, d);
3422 /*                      if (iptr->dst->flags & INMEMORY) { */
3423 /*                              x86_64_mov_reg_membase(cd, s1, REG_SP, iptr->dst->regoff * 8); */
3424 /*                      } else { */
3425 /*                              M_INTMOVE(s1, iptr->dst->regoff); */
3426 /*                      } */
3427                         }
3428                         break;
3429
3430                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3431
3432                                       /* op1:   0 == array, 1 == class                */
3433                                       /* val.a: (classinfo *) superclass              */
3434
3435                         /*  superclass is an interface:
3436                          *
3437                          *  return (sub != NULL) &&
3438                          *         (sub->vftbl->interfacetablelength > super->index) &&
3439                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3440                          *
3441                          *  superclass is a class:
3442                          *
3443                          *  return ((sub != NULL) && (0
3444                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3445                          *          super->vftbl->diffvall));
3446                          */
3447
3448                         {
3449                         classinfo *super;
3450                         vftbl_t   *supervftbl;
3451                         s4         superindex;
3452
3453                         super = (classinfo *) iptr->val.a;
3454
3455                         if (!super) {
3456                                 superindex = 0;
3457                                 supervftbl = NULL;
3458
3459                         } else {
3460                                 superindex = super->index;
3461                                 supervftbl = super->vftbl;
3462                         }
3463
3464 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3465             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3466 #endif
3467
3468                         var_to_reg_int(s1, src, REG_ITMP1);
3469                         d = reg_of_var(rd, iptr->dst, REG_ITMP2);
3470                         if (s1 == d) {
3471                                 M_INTMOVE(s1, REG_ITMP1);
3472                                 s1 = REG_ITMP1;
3473                         }
3474
3475                         /* calculate interface instanceof code size */
3476
3477                         s2 = 3; /* mov_membase_reg */
3478                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3479                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3480                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3481                                 3 /* test */ + 4 /* setcc */;
3482
3483                         if (!super)
3484                                 s2 += (showdisassemble ? 5 : 0);
3485
3486                         /* calculate class instanceof code size */
3487
3488                         s3 = 3; /* mov_membase_reg */
3489                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3490                         s3 += 10; /* mov_imm_reg */
3491                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3492                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3493                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3494                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3495                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3496                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3497                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3498
3499                         if (!super)
3500                                 s3 += (showdisassemble ? 5 : 0);
3501
3502                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3503
3504                         /* if class is not resolved, check which code to call */
3505
3506                         if (!super) {
3507                                 x86_64_test_reg_reg(cd, s1, s1);
3508                                 x86_64_jcc(cd, X86_64_CC_Z, (6 + (showdisassemble ? 5 : 0) +
3509                                                                                          7 + 6 + s2 + 5 + s3));
3510
3511                                 codegen_addpatchref(cd, cd->mcodeptr,
3512                                                                         PATCHER_checkcast_instanceof_flags,
3513                                                                         (constant_classref *) iptr->target);
3514
3515                                 if (showdisassemble) {
3516                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3517                                 }
3518
3519                                 x86_64_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3520                                 x86_64_alul_imm_reg(cd, X86_64_AND, ACC_INTERFACE, REG_ITMP3);
3521                                 x86_64_jcc(cd, X86_64_CC_Z, s2 + 5);
3522                         }
3523
3524                         /* interface instanceof code */
3525
3526                         if (!super || (super->flags & ACC_INTERFACE)) {
3527                                 if (super) {
3528                                         x86_64_test_reg_reg(cd, s1, s1);
3529                                         x86_64_jcc(cd, X86_64_CC_Z, s2);
3530                                 }
3531
3532                                 x86_64_mov_membase_reg(cd, s1,
3533                                                                            OFFSET(java_objectheader, vftbl),
3534                                                                            REG_ITMP1);
3535                                 if (!super) {
3536                                         codegen_addpatchref(cd, cd->mcodeptr,
3537                                                                                 PATCHER_checkcast_instanceof_interface,
3538                                                                                 (constant_classref *) iptr->target);
3539
3540                                         if (showdisassemble) {
3541                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3542                                         }
3543                                 }
3544
3545                                 x86_64_movl_membase32_reg(cd, REG_ITMP1,
3546                                                                                   OFFSET(vftbl_t, interfacetablelength),
3547                                                                                   REG_ITMP3);
3548                                 x86_64_alu_imm32_reg(cd, X86_64_SUB, superindex, REG_ITMP3);
3549                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3550
3551                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3552
3553                                 x86_64_jcc(cd, X86_64_CC_LE, a);
3554                                 x86_64_mov_membase32_reg(cd, REG_ITMP1,
3555                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3556                                                                                  superindex * sizeof(methodptr*),
3557                                                                                  REG_ITMP1);
3558                                 x86_64_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3559                                 x86_64_setcc_reg(cd, X86_64_CC_NE, d);
3560
3561                                 if (!super)
3562                                         x86_64_jmp_imm(cd, s3);
3563                         }
3564
3565                         /* class instanceof code */
3566
3567                         if (!super || !(super->flags & ACC_INTERFACE)) {
3568                                 if (super) {
3569                                         x86_64_test_reg_reg(cd, s1, s1);
3570                                         x86_64_jcc(cd, X86_64_CC_E, s3);
3571                                 }
3572
3573                                 x86_64_mov_membase_reg(cd, s1,
3574                                                                            OFFSET(java_objectheader, vftbl),
3575                                                                            REG_ITMP1);
3576
3577                                 if (!super) {
3578                                         codegen_addpatchref(cd, cd->mcodeptr,
3579                                                                                 PATCHER_instanceof_class,
3580                                                                                 (constant_classref *) iptr->target);
3581
3582                                         if (showdisassemble) {
3583                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3584                                         }
3585                                 }
3586
3587                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3588 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3589                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3590 #endif
3591                                 x86_64_movl_membase_reg(cd, REG_ITMP1,
3592                                                                                 OFFSET(vftbl_t, baseval),
3593                                                                                 REG_ITMP1);
3594                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3595                                                                                 OFFSET(vftbl_t, diffval),
3596                                                                                 REG_ITMP3);
3597                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3598                                                                                 OFFSET(vftbl_t, baseval),
3599                                                                                 REG_ITMP2);
3600 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3601                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3602 #endif
3603                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
3604                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d); /* may be REG_ITMP2 */
3605                                 x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP3, REG_ITMP1);
3606                                 x86_64_setcc_reg(cd, X86_64_CC_BE, d);
3607                         }
3608                         store_reg_to_var_int(iptr->dst, d);
3609                         }
3610                         break;
3611
3612                 case ICMD_CHECKASIZE:  /* ..., size ==> ..., size                     */
3613
3614                         if (src->flags & INMEMORY) {
3615                                 x86_64_alul_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
3616
3617                         } else {
3618                                 x86_64_testl_reg_reg(cd, src->regoff, src->regoff);
3619                         }
3620                         x86_64_jcc(cd, X86_64_CC_L, 0);
3621                         codegen_addxcheckarefs(cd, cd->mcodeptr);
3622                         break;
3623
3624                 case ICMD_CHECKEXCEPTION:    /* ... ==> ...                           */
3625
3626                         x86_64_test_reg_reg(cd, REG_RESULT, REG_RESULT);
3627                         x86_64_jcc(cd, X86_64_CC_E, 0);
3628                         codegen_addxexceptionrefs(cd, cd->mcodeptr);
3629                         break;
3630
3631                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3632                                          /* op1 = dimension, val.a = array descriptor */
3633
3634                         /* check for negative sizes and copy sizes to stack if necessary  */
3635
3636                         MCODECHECK((iptr->op1 << 1) + 64);
3637
3638                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3639                                 var_to_reg_int(s2, src, REG_ITMP1);
3640                                 x86_64_testl_reg_reg(cd, s2, s2);
3641                                 x86_64_jcc(cd, X86_64_CC_L, 0);
3642                                 codegen_addxcheckarefs(cd, cd->mcodeptr);
3643
3644                                 /* copy SAVEDVAR sizes to stack */
3645
3646                                 if (src->varkind != ARGVAR) {
3647                                         x86_64_mov_reg_membase(cd, s2, REG_SP, s1 * 8);
3648                                 }
3649                         }
3650
3651                         /* is a patcher function set? */
3652
3653                         if (iptr->target) {
3654                                 codegen_addpatchref(cd, cd->mcodeptr,
3655                                                                         (functionptr) (ptrint) iptr->target,
3656                                                                         iptr->val.a);
3657
3658                                 if (showdisassemble) {
3659                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3660                                 }
3661
3662                                 a = 0;
3663
3664                         } else {
3665                                 a = (ptrint) iptr->val.a;
3666                         }
3667
3668                         /* a0 = dimension count */
3669
3670                         x86_64_mov_imm_reg(cd, iptr->op1, rd->argintregs[0]);
3671
3672                         /* a1 = arrayvftbl */
3673
3674                         x86_64_mov_imm_reg(cd, (ptrint) iptr->val.a, rd->argintregs[1]);
3675
3676                         /* a2 = pointer to dimensions = stack pointer */
3677
3678                         x86_64_mov_reg_reg(cd, REG_SP, rd->argintregs[2]);
3679
3680                         x86_64_mov_imm_reg(cd, (ptrint) BUILTIN_multianewarray, REG_ITMP1);
3681                         x86_64_call_reg(cd, REG_ITMP1);
3682
3683                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3684                         M_INTMOVE(REG_RESULT, s1);
3685                         store_reg_to_var_int(iptr->dst, s1);
3686                         break;
3687
3688                 default:
3689                         throw_cacao_exception_exit(string_java_lang_InternalError,
3690                                                                            "Unknown ICMD %d", iptr->opc);
3691         } /* switch */
3692
3693         } /* for instruction */
3694
3695         /* copy values to interface registers */
3696
3697         src = bptr->outstack;
3698         len = bptr->outdepth;
3699         MCODECHECK(64 + len);
3700 #ifdef LSRA
3701         if (!opt_lsra)
3702 #endif
3703         while (src) {
3704                 len--;
3705                 if ((src->varkind != STACKVAR)) {
3706                         s2 = src->type;
3707                         if (IS_FLT_DBL_TYPE(s2)) {
3708                                 var_to_reg_flt(s1, src, REG_FTMP1);
3709                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3710                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3711
3712                                 } else {
3713                                         x86_64_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3714                                 }
3715
3716                         } else {
3717                                 var_to_reg_int(s1, src, REG_ITMP1);
3718                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3719                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3720
3721                                 } else {
3722                                         x86_64_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3723                                 }
3724                         }
3725                 }
3726                 src = src->prev;
3727         }
3728         } /* if (bptr -> flags >= BBREACHED) */
3729         } /* for basic block */
3730
3731         codegen_createlinenumbertable(cd);
3732
3733         {
3734
3735         /* generate bound check stubs */
3736
3737         u1 *xcodeptr = NULL;
3738         branchref *bref;
3739
3740         for (bref = cd->xboundrefs; bref != NULL; bref = bref->next) {
3741                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3742                                   bref->branchpos,
3743                                                   cd->mcodeptr - cd->mcodebase);
3744
3745                 MCODECHECK(100);
3746
3747                 /* move index register into REG_ITMP1 */
3748                 x86_64_mov_reg_reg(cd, bref->reg, REG_ITMP1);             /* 3 bytes  */
3749
3750                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                 /* 10 bytes */
3751                 dseg_adddata(cd, cd->mcodeptr);
3752                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);   /* 10 bytes */
3753                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC); /* 3 bytes  */
3754
3755                 if (xcodeptr != NULL) {
3756                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3757
3758                 } else {
3759                         xcodeptr = cd->mcodeptr;
3760
3761
3762                         /*create stackinfo -- begin*/
3763                         x86_64_alu_imm_reg(cd, X86_64_SUB, 4 * 8, REG_SP);
3764                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 3 * 8);
3765                         x86_64_mov_imm_membase(cd,0,REG_SP,2*8);
3766                         x86_64_mov_imm_membase(cd,0,REG_SP,1*8);
3767                         x86_64_mov_imm_reg(cd,(ptrint)asm_prepare_native_stackinfo,REG_ITMP3);
3768                         x86_64_call_reg(cd,REG_ITMP3);
3769                         /*create stackinfo -- end*/
3770
3771                         x86_64_mov_reg_reg(cd, REG_ITMP1, rd->argintregs[0]);
3772                         x86_64_mov_imm_reg(cd, (ptrint) new_arrayindexoutofboundsexception, REG_ITMP3);
3773                         x86_64_call_reg(cd, REG_ITMP3);
3774
3775                         /*remove stackinfo -- begin*/
3776                         x86_64_mov_imm_reg(cd,(ptrint)asm_remove_native_stackinfo,REG_ITMP3);
3777                         x86_64_call_reg(cd,REG_ITMP3);
3778                         /*remove stackinfo -- end*/
3779
3780                         x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, REG_ITMP2_XPC);
3781                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3782
3783                         x86_64_mov_imm_reg(cd, (ptrint) asm_handle_exception, REG_ITMP3);
3784                         x86_64_jmp_reg(cd, REG_ITMP3);
3785                 }
3786         }
3787
3788         /* generate negative array size check stubs */
3789
3790         xcodeptr = NULL;
3791
3792         for (bref = cd->xcheckarefs; bref != NULL; bref = bref->next) {
3793                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3794                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3795                                                           bref->branchpos,
3796                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3797                         continue;
3798                 }
3799
3800                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3801                                   bref->branchpos,
3802                                                   cd->mcodeptr - cd->mcodebase);
3803
3804                 MCODECHECK(100);
3805
3806                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                 /* 10 bytes */
3807                 dseg_adddata(cd, cd->mcodeptr);
3808                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);   /* 10 bytes */
3809                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC); /* 3 bytes  */
3810
3811                 if (xcodeptr != NULL) {
3812                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3813
3814                 } else {
3815                         xcodeptr = cd->mcodeptr;
3816
3817
3818                         /*create stackinfo -- begin*/
3819                         x86_64_alu_imm_reg(cd, X86_64_SUB, 4 * 8, REG_SP);
3820                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 3 * 8);
3821                         x86_64_mov_imm_membase(cd,0,REG_SP,2*8);
3822                         x86_64_mov_imm_membase(cd,0,REG_SP,1*8);
3823                         x86_64_mov_imm_reg(cd,(ptrint)asm_prepare_native_stackinfo,REG_ITMP3);
3824                         x86_64_call_reg(cd,REG_ITMP3);
3825                         /*create stackinfo -- end*/
3826
3827                         x86_64_mov_imm_reg(cd, (u8) new_negativearraysizeexception, REG_ITMP3);
3828                         x86_64_call_reg(cd, REG_ITMP3);
3829
3830                         /*remove stackinfo -- begin*/
3831                         x86_64_mov_imm_reg(cd,(ptrint)asm_remove_native_stackinfo,REG_ITMP3);
3832                         x86_64_call_reg(cd,REG_ITMP3);
3833                         /*remove stackinfo -- end*/
3834
3835                         x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, REG_ITMP2_XPC);
3836                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3837
3838                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
3839                         x86_64_jmp_reg(cd, REG_ITMP3);
3840                 }
3841         }
3842
3843         /* generate cast check stubs */
3844
3845         xcodeptr = NULL;
3846
3847         for (bref = cd->xcastrefs; bref != NULL; bref = bref->next) {
3848                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3849                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3850                                                           bref->branchpos,
3851                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3852                         continue;
3853                 }
3854
3855                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3856                                   bref->branchpos,
3857                                                   cd->mcodeptr - cd->mcodebase);
3858
3859                 MCODECHECK(100);
3860
3861                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                 /* 10 bytes */
3862                 dseg_adddata(cd, cd->mcodeptr);
3863                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);   /* 10 bytes */
3864                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC); /* 3 bytes  */
3865
3866                 if (xcodeptr != NULL) {
3867                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3868
3869                 } else {
3870                         xcodeptr = cd->mcodeptr;
3871
3872                         /*create stackinfo -- begin*/
3873                         x86_64_alu_imm_reg(cd, X86_64_SUB, 4 * 8, REG_SP);
3874                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 3 * 8);
3875                         x86_64_mov_imm_membase(cd,0,REG_SP,2*8);
3876                         x86_64_mov_imm_membase(cd,0,REG_SP,1*8);
3877                         x86_64_mov_imm_reg(cd,(ptrint)asm_prepare_native_stackinfo,REG_ITMP3);
3878                         x86_64_call_reg(cd,REG_ITMP3);
3879                         /*create stackinfo -- end*/
3880
3881
3882                         x86_64_mov_imm_reg(cd, (u8) new_classcastexception, REG_ITMP3);
3883                         x86_64_call_reg(cd, REG_ITMP3);
3884
3885                         /*remove stackinfo -- begin*/
3886                         x86_64_mov_imm_reg(cd,(ptrint)asm_remove_native_stackinfo,REG_ITMP3);
3887                         x86_64_call_reg(cd,REG_ITMP3);
3888                         /*remove stackinfo -- end*/
3889
3890                         x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, REG_ITMP2_XPC);
3891                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3892
3893                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
3894                         x86_64_jmp_reg(cd, REG_ITMP3);
3895                 }
3896         }
3897
3898         /* generate divide by zero check stubs */
3899
3900         xcodeptr = NULL;
3901
3902         for (bref = cd->xdivrefs; bref != NULL; bref = bref->next) {
3903                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3904                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3905                                                           bref->branchpos,
3906                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3907                         continue;
3908                 }
3909
3910                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3911                                   bref->branchpos,
3912                                                   cd->mcodeptr - cd->mcodebase);
3913
3914                 MCODECHECK(100);
3915
3916                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                 /* 10 bytes */
3917                 dseg_adddata(cd, cd->mcodeptr);
3918                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);   /* 10 bytes */
3919                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC); /* 3 bytes  */
3920
3921                 if (xcodeptr != NULL) {
3922                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3923
3924                 } else {
3925                         xcodeptr = cd->mcodeptr;
3926
3927                         /*create stackinfo -- begin*/
3928                         x86_64_alu_imm_reg(cd, X86_64_SUB, 4 * 8, REG_SP);
3929                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 3 * 8);
3930                         x86_64_mov_imm_membase(cd,0,REG_SP,2*8);
3931                         x86_64_mov_imm_membase(cd,0,REG_SP,1*8);
3932                         x86_64_mov_imm_reg(cd,(ptrint)asm_prepare_native_stackinfo,REG_ITMP3);
3933                         x86_64_call_reg(cd,REG_ITMP3);
3934                         /*create stackinfo -- end*/
3935
3936                         x86_64_mov_imm_reg(cd, (u8) new_arithmeticexception, REG_ITMP3);
3937                         x86_64_call_reg(cd, REG_ITMP3);
3938
3939                         /*remove stackinfo -- begin*/
3940                         x86_64_mov_imm_reg(cd,(ptrint)asm_remove_native_stackinfo,REG_ITMP3);
3941                         x86_64_call_reg(cd,REG_ITMP3);
3942                         /*remove stackinfo -- end*/
3943
3944                         x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, REG_ITMP2_XPC);
3945                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3946
3947                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
3948                         x86_64_jmp_reg(cd, REG_ITMP3);
3949                 }
3950         }
3951
3952         /* generate exception check stubs */
3953
3954         xcodeptr = NULL;
3955
3956         for (bref = cd->xexceptionrefs; bref != NULL; bref = bref->next) {
3957                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3958                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3959                                                           bref->branchpos,
3960                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3961                         continue;
3962                 }
3963
3964                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3965                                   bref->branchpos,
3966                                                   cd->mcodeptr - cd->mcodebase);
3967
3968                 MCODECHECK(100);
3969
3970                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                 /* 10 bytes */
3971                 dseg_adddata(cd, cd->mcodeptr);
3972                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP1);   /* 10 bytes */
3973                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP1, REG_ITMP2_XPC); /* 3 bytes  */
3974
3975                 if (xcodeptr != NULL) {
3976                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3977
3978                 } else {
3979                         xcodeptr = cd->mcodeptr;
3980
3981
3982
3983                         x86_64_alu_imm_reg(cd, X86_64_SUB, 4*8, REG_SP);
3984                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 3*8);
3985                         x86_64_mov_imm_membase(cd, 0, REG_SP, 2*8);
3986                         x86_64_mov_imm_membase(cd, 0, REG_SP, 1*8);
3987                         x86_64_mov_imm_membase(cd, 0, REG_SP, 0*8);
3988                         x86_64_mov_imm_reg(cd,(u8) asm_prepare_native_stackinfo,REG_ITMP1);
3989                         x86_64_call_reg(cd,REG_ITMP1);
3990
3991
3992 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3993                         x86_64_mov_imm_reg(cd, (u8) &builtin_get_exceptionptrptr, REG_ITMP1);
3994                         x86_64_call_reg(cd, REG_ITMP1);
3995                         x86_64_mov_membase_reg(cd, REG_RESULT, 0, REG_ITMP3);
3996                         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
3997                         x86_64_mov_reg_reg(cd, REG_ITMP3, REG_ITMP1_XPTR);
3998 #else
3999                         x86_64_mov_imm_reg(cd, (u8) &_exceptionptr, REG_ITMP3);
4000                         x86_64_mov_membase_reg(cd, REG_ITMP3, 0, REG_ITMP1_XPTR);
4001                         x86_64_mov_imm_membase(cd, 0, REG_ITMP3, 0);
4002 #endif
4003                         x86_64_mov_reg_reg(cd,REG_ITMP1_XPTR,RDI);
4004                         x86_64_mov_imm_reg(cd,(u8) helper_fillin_stacktrace_always,REG_ITMP1);
4005                         x86_64_call_reg(cd,REG_ITMP1);
4006                         x86_64_mov_reg_reg(cd,REG_RESULT,REG_ITMP1_XPTR);
4007
4008                         x86_64_mov_imm_reg(cd,(u8) asm_remove_native_stackinfo,REG_ITMP2);
4009                         x86_64_call_reg(cd,REG_ITMP2);
4010
4011                         x86_64_alu_imm_reg(cd, X86_64_ADD, 8, REG_SP);
4012                         x86_64_mov_membase_reg(cd, REG_SP, 0, REG_ITMP2_XPC);
4013                         x86_64_alu_imm_reg(cd, X86_64_ADD, 8, REG_SP);
4014
4015
4016                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
4017                         x86_64_jmp_reg(cd, REG_ITMP3);
4018                 }
4019         }
4020
4021         /* generate null pointer check stubs */
4022
4023         xcodeptr = NULL;
4024
4025         for (bref = cd->xnullrefs; bref != NULL; bref = bref->next) {
4026                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
4027                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
4028                                                           bref->branchpos,
4029                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
4030                         continue;
4031                 }
4032
4033                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
4034                                   bref->branchpos,
4035                                                   cd->mcodeptr - cd->mcodebase);
4036
4037                 MCODECHECK(100);
4038
4039                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                 /* 10 bytes */
4040                 dseg_adddata(cd, cd->mcodeptr);
4041                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP1);   /* 10 bytes */
4042                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP1, REG_ITMP2_XPC); /* 3 bytes  */
4043
4044                 if (xcodeptr != NULL) {
4045                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
4046
4047                 } else {
4048                         xcodeptr = cd->mcodeptr;
4049
4050                         /*create stackinfo -- begin*/
4051                         x86_64_alu_imm_reg(cd, X86_64_SUB, 4 * 8, REG_SP);
4052                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 3 * 8);
4053                         x86_64_mov_imm_membase(cd,0,REG_SP,2*8);
4054                         x86_64_mov_imm_membase(cd,0,REG_SP,1*8);
4055                         x86_64_mov_imm_reg(cd,(ptrint)asm_prepare_native_stackinfo,REG_ITMP3);
4056                         x86_64_call_reg(cd,REG_ITMP3);
4057                         /*create stackinfo -- end*/
4058
4059
4060                         x86_64_mov_imm_reg(cd, (ptrint) new_nullpointerexception, REG_ITMP3);
4061                         x86_64_call_reg(cd, REG_ITMP3);
4062
4063                         /*remove stackinfo -- begin*/
4064                         x86_64_mov_imm_reg(cd,(ptrint)asm_remove_native_stackinfo,REG_ITMP3);
4065                         x86_64_call_reg(cd,REG_ITMP3);
4066                         /*remove stackinfo -- end*/
4067
4068                         x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, REG_ITMP2_XPC);
4069                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
4070
4071                         x86_64_mov_imm_reg(cd, (ptrint) asm_handle_exception, REG_ITMP3);
4072                         x86_64_jmp_reg(cd, REG_ITMP3);
4073                 }
4074         }
4075
4076         /* generate code patching stub call code */
4077
4078         {
4079                 patchref    *pref;
4080                 codegendata *tmpcd;
4081                 ptrint       mcode;
4082
4083                 tmpcd = DNEW(codegendata);
4084
4085                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4086                         /* check size of code segment */
4087
4088                         MCODECHECK(128);
4089
4090                         /* Get machine code which is patched back in later. A             */
4091                         /* `call rel32' is 5 bytes long (but read 8 bytes).               */
4092
4093                         xcodeptr = cd->mcodebase + pref->branchpos;
4094                         mcode = *((ptrint *) xcodeptr);
4095
4096                         /* patch in `call rel32' to call the following code               */
4097
4098                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
4099                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
4100
4101                         /* move pointer to java_objectheader onto stack */
4102
4103 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4104                         /* create a virtual java_objectheader */
4105
4106                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4107                         a = dseg_addaddress(cd, NULL);                      /* vftbl      */
4108
4109                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP3);
4110                         x86_64_push_reg(cd, REG_ITMP3);
4111 #else
4112                         x86_64_push_imm(cd, 0);
4113 #endif
4114
4115                         /* move machine code bytes and classinfo pointer into registers */
4116
4117                         x86_64_mov_imm_reg(cd, (ptrint) mcode, REG_ITMP3);
4118                         x86_64_push_reg(cd, REG_ITMP3);
4119                         x86_64_mov_imm_reg(cd, (ptrint) pref->ref, REG_ITMP3);
4120                         x86_64_push_reg(cd, REG_ITMP3);
4121
4122                         x86_64_mov_imm_reg(cd, (ptrint) pref->patcher, REG_ITMP3);
4123                         x86_64_push_reg(cd, REG_ITMP3);
4124
4125                         x86_64_mov_imm_reg(cd, (ptrint) asm_wrapper_patcher, REG_ITMP3);
4126                         x86_64_jmp_reg(cd, REG_ITMP3);
4127                 }
4128         }
4129         }
4130
4131         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
4132 }
4133
4134
4135 /* createcompilerstub **********************************************************
4136
4137    Creates a stub routine which calls the compiler.
4138
4139 *******************************************************************************/
4140
4141 #define COMPILERSTUB_SIZE    23
4142
4143 functionptr createcompilerstub(methodinfo *m)
4144 {
4145         u1          *s;                     /* memory to hold the stub            */
4146         codegendata *cd;
4147         s4           dumpsize;
4148
4149         s = CNEW(u1, COMPILERSTUB_SIZE);
4150
4151         /* mark start of dump memory area */
4152
4153         dumpsize = dump_size();
4154
4155         cd = DNEW(codegendata);
4156         cd->mcodeptr = s;
4157
4158         /* code for the stub */
4159
4160         x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP1); /* pass method to compiler */
4161         x86_64_mov_imm_reg(cd, (ptrint) asm_call_jit_compiler, REG_ITMP3);
4162         x86_64_jmp_reg(cd, REG_ITMP3);
4163
4164 #if defined(STATISTICS)
4165         if (opt_stat)
4166                 count_cstub_len += COMPILERSTUB_SIZE;
4167 #endif
4168
4169         /* release dump area */
4170
4171         dump_release(dumpsize);
4172
4173         return (functionptr) (ptrint) s;
4174 }
4175
4176
4177 /* createnativestub ************************************************************
4178
4179    Creates a stub routine which calls a native method.
4180
4181 *******************************************************************************/
4182
4183 functionptr createnativestub(functionptr f, methodinfo *m, codegendata *cd, registerdata *rd)
4184 {
4185         methoddesc         *md;
4186         methoddesc         *nmd;
4187         s4                  stackframesize; /* size of stackframe if needed       */
4188         s4                  nativeparams;
4189         s4                  i, j;           /* count variables                    */
4190         s4                  t;
4191         s4                  s1, s2, off;
4192
4193         /* initialize variables */
4194
4195         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
4196
4197         /* create new method descriptor with additional native parameters */
4198
4199         md = m->parseddesc;
4200
4201         nmd = (methoddesc *) DMNEW(u1, sizeof(methoddesc) - sizeof(typedesc) +
4202                                                            md->paramcount * sizeof(typedesc) +
4203                                                            nativeparams * sizeof(typedesc));
4204
4205         nmd->paramcount = md->paramcount + nativeparams;
4206
4207         nmd->params = DMNEW(paramdesc, nmd->paramcount);
4208
4209         nmd->paramtypes[0].type = TYPE_ADR; /* add environment pointer            */
4210
4211         if (m->flags & ACC_STATIC)
4212                 nmd->paramtypes[1].type = TYPE_ADR; /* add class pointer              */
4213
4214         MCOPY(nmd->paramtypes + nativeparams, md->paramtypes, typedesc,
4215                   md->paramcount);
4216
4217         md_param_alloc(nmd);
4218
4219
4220         /* create method header */
4221
4222         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
4223         (void) dseg_adds4(cd, 0 * 8);                           /* FrameSize      */
4224         (void) dseg_adds4(cd, 0);                               /* IsSync         */
4225         (void) dseg_adds4(cd, 0);                               /* IsLeaf         */
4226         (void) dseg_adds4(cd, 0);                               /* IntSave        */
4227         (void) dseg_adds4(cd, 0);                               /* FltSave        */
4228         (void) dseg_addlinenumbertablesize(cd);
4229         (void) dseg_adds4(cd, 0);                               /* ExTableSize    */
4230
4231
4232         /* initialize mcode variables */
4233
4234         cd->mcodeptr = (u1 *) cd->mcodebase;
4235         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
4236
4237
4238         /* if function is static, check for initialized */
4239
4240         if ((m->flags & ACC_STATIC) && !m->class->initialized) {
4241                 codegen_addpatchref(cd, cd->mcodeptr, PATCHER_clinit, m->class);
4242
4243                 if (showdisassemble) {
4244                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
4245                 }
4246         }
4247
4248         if (runverbose) {
4249                 s4 l, s1;
4250
4251                 M_ASUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + 1) * 8, REG_SP);
4252
4253                 /* save integer and float argument registers */
4254
4255                 for (i = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4256                         M_LST(rd->argintregs[i], REG_SP, (1 + i) * 8);
4257
4258                 for (i = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4259                         M_DST(rd->argfltregs[i], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4260
4261                 /* show integer hex code for float arguments */
4262
4263                 for (i = 0, l = 0; i < md->paramcount && i < INT_ARG_CNT; i++) {
4264                         /* if the paramtype is a float, we have to right shift all        */
4265                         /* following integer registers                                    */
4266
4267                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type)) {
4268                                 for (s1 = INT_ARG_CNT - 2; s1 >= i; s1--)
4269                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
4270
4271                                 x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[i]);
4272                                 l++;
4273                         }
4274                 }
4275
4276                 x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP1);
4277                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, 0 * 8);
4278                 x86_64_mov_imm_reg(cd, (ptrint) builtin_trace_args, REG_ITMP1);
4279                 x86_64_call_reg(cd, REG_ITMP1);
4280
4281                 /* restore integer and float argument registers */
4282
4283                 for (i = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4284                         M_LLD(rd->argintregs[i], REG_SP, (1 + i) * 8);
4285
4286                 for (i = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4287                         M_DLD(rd->argfltregs[i], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4288
4289                 M_AADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + 1) * 8, REG_SP);
4290         }
4291
4292         /* 4 == additional size needed for native stack frame information*/
4293
4294         M_ASUB_IMM((4 + INT_ARG_CNT + FLT_ARG_CNT + 1) * 8, REG_SP);
4295
4296         /* save integer and float argument registers */
4297
4298         for (i = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4299                 M_LST(rd->argintregs[i], REG_SP, i * 8);
4300
4301         for (i = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4302                 M_DST(rd->argfltregs[i], REG_SP, (INT_ARG_CNT + i) * 8);
4303
4304 /*
4305 15+
4306 0*8        void *oldThreadspecificHeadValue;
4307 1*8        void **addressOfThreadspecificHead;
4308 2*8        methodinfo *method;
4309 3*8        void *beginOfJavaStackframe; only used if != 0
4310 4*8        void *returnToFromNative;
4311 */
4312
4313         /* CREATE DYNAMIC STACK INFO -- BEGIN   offsets:15,16,17,18*/
4314
4315         x86_64_mov_imm_membase(cd, 0, REG_SP, 18 * 8);
4316         x86_64_mov_imm_membase(cd, (ptrint) m, REG_SP, 17 * 8);
4317
4318         x86_64_mov_imm_reg(cd, (ptrint) builtin_asm_get_stackframeinfo, REG_ITMP1);
4319         x86_64_call_reg(cd, REG_ITMP1);
4320
4321         x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 16 * 8);
4322         x86_64_mov_membase_reg(cd, REG_RESULT, 0, REG_ITMP2);
4323         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_SP, 15 * 8);
4324         x86_64_mov_reg_reg(cd, REG_SP, REG_ITMP2);
4325         x86_64_alu_imm_reg(cd, X86_64_ADD, (1 + INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_ITMP2);
4326         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_RESULT, 0);
4327
4328         /* CREATE DYNAMIC STACK INFO -- END*/
4329
4330         STATS({
4331                 x86_64_mov_imm_reg(cd,(u8)nativeinvokation,REG_ITMP1);
4332                 x86_64_call_reg(cd,REG_ITMP1);
4333         })
4334
4335         /* restore integer and float argument registers */
4336
4337         for (i = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4338                 M_LLD(rd->argintregs[i], REG_SP, i * 8);
4339
4340         for (i = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4341                 M_DLD(rd->argfltregs[i], REG_SP, (INT_ARG_CNT + i) * 8);
4342
4343         M_AADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + 1) * 8, REG_SP);
4344
4345
4346         /* calculate stack frame size */
4347
4348         stackframesize = nmd->memuse;
4349
4350         /* keep stack 16-byte aligned */
4351         if (!(stackframesize & 0x1))
4352                 stackframesize++;
4353
4354         M_ASUB_IMM(stackframesize * 8, REG_SP);
4355
4356         /* copy or spill arguments to new locations */
4357
4358         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
4359                 t = md->paramtypes[i].type;
4360
4361                 if (IS_INT_LNG_TYPE(t)) {
4362                         if (!md->params[i].inmemory) {
4363                                 s1 = rd->argintregs[md->params[i].regoff];
4364
4365                                 if (!nmd->params[j].inmemory) {
4366                                         s2 = rd->argintregs[nmd->params[j].regoff];
4367                                         M_INTMOVE(s1, s2);
4368
4369                                 } else {
4370                                         s2 = nmd->params[j].regoff;
4371                                         M_LST(s1, REG_SP, s2 * 8);
4372                                 }
4373
4374                         } else {
4375                                 s1 = md->params[i].regoff + stackframesize;
4376                                 s2 = nmd->params[j].regoff;
4377                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
4378                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
4379                         }
4380
4381                 } else {
4382                         /* We only copy spilled float arguments, as the float argument    */
4383                         /* registers keep unchanged.                                      */
4384
4385                         if (md->params[i].inmemory) {
4386                                 s1 = md->params[i].regoff + stackframesize;
4387                                 s2 = nmd->params[j].regoff;
4388                                 M_DLD(REG_FTMP1, REG_SP, s1 * 8);
4389                                 M_DST(REG_FTMP1, REG_SP, s2 * 8);
4390                         }
4391                 }
4392         }
4393
4394         /* put class into second argument register */
4395
4396         if (m->flags & ACC_STATIC)
4397                 x86_64_mov_imm_reg(cd, (ptrint) m->class, rd->argintregs[1]);
4398
4399         /* put env into first argument register */
4400
4401         x86_64_mov_imm_reg(cd, (ptrint) &env, rd->argintregs[0]);
4402
4403         /* do the native function call */
4404
4405 #if !defined(STATIC_CLASSPATH)
4406         if (f == NULL) {
4407                 codegen_addpatchref(cd, cd->mcodeptr, PATCHER_resolve_native, m);
4408
4409                 if (showdisassemble) {
4410                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
4411                 }
4412         }
4413 #endif
4414
4415         x86_64_mov_imm_reg(cd, (ptrint) f, REG_ITMP1);
4416         x86_64_call_reg(cd, REG_ITMP1);
4417
4418
4419         /* remove stackframe if there is one */
4420
4421         if (stackframesize)
4422                 M_AADD_IMM(stackframesize * 8, REG_SP);
4423
4424         /* remove dynamic stack info */
4425
4426         x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 2 * 8);
4427         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2);
4428         x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, REG_RESULT);
4429         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_RESULT, 0);
4430         x86_64_mov_membase_reg(cd, REG_SP, 2 * 8, REG_RESULT);
4431
4432         M_AADD_IMM(4 * 8, REG_SP);
4433
4434         /* generate call trace */
4435
4436         if (runverbose) {
4437                 M_ASUB_IMM(3 * 8, REG_SP);              /* keep stack 16-byte aligned */
4438
4439                 M_LST(REG_RESULT, REG_SP, 0 * 8);
4440                 M_DST(REG_FRESULT, REG_SP, 1 * 8);
4441
4442                 x86_64_mov_imm_reg(cd, (ptrint) m, rd->argintregs[0]);
4443                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
4444                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4445                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4446
4447                 x86_64_mov_imm_reg(cd, (ptrint) builtin_displaymethodstop, REG_ITMP1);
4448                 x86_64_call_reg(cd, REG_ITMP1);
4449
4450                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
4451                 M_DLD(REG_FRESULT, REG_SP, 1 * 8);
4452
4453                 M_AADD_IMM(3 * 8, REG_SP);
4454         }
4455
4456         /* check for exception */
4457
4458 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4459         x86_64_push_reg(cd, REG_RESULT);
4460         x86_64_mov_imm_reg(cd, (ptrint) builtin_get_exceptionptrptr, REG_ITMP3);
4461         x86_64_call_reg(cd, REG_ITMP3);
4462         x86_64_mov_membase_reg(cd, REG_RESULT, 0, REG_ITMP3);
4463         x86_64_pop_reg(cd, REG_RESULT);
4464 #else
4465         x86_64_mov_imm_reg(cd, (ptrint) &_exceptionptr, REG_ITMP3);
4466         x86_64_mov_membase_reg(cd, REG_ITMP3, 0, REG_ITMP3);
4467 #endif
4468         x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
4469         x86_64_jcc(cd, X86_64_CC_NE, 1);
4470
4471         x86_64_ret(cd);
4472
4473         /* handle exception */
4474
4475 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4476         x86_64_push_reg(cd, REG_ITMP3);
4477         x86_64_mov_imm_reg(cd, (ptrint) builtin_get_exceptionptrptr, REG_ITMP3);
4478         x86_64_call_reg(cd, REG_ITMP3);
4479         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
4480         x86_64_pop_reg(cd, REG_ITMP1_XPTR);
4481 #else
4482         x86_64_mov_reg_reg(cd, REG_ITMP3, REG_ITMP1_XPTR);
4483         x86_64_mov_imm_reg(cd, (ptrint) &_exceptionptr, REG_ITMP3);
4484         x86_64_alu_reg_reg(cd, X86_64_XOR, REG_ITMP2, REG_ITMP2);
4485         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_ITMP3, 0);    /* clear exception pointer */
4486 #endif
4487
4488         x86_64_mov_membase_reg(cd, REG_SP, 0, REG_ITMP2_XPC);    /* get return address from stack */
4489         x86_64_alu_imm_reg(cd, X86_64_SUB, 3, REG_ITMP2_XPC);    /* callq */
4490
4491         x86_64_mov_imm_reg(cd, (ptrint) asm_handle_nat_exception, REG_ITMP3);
4492         x86_64_jmp_reg(cd, REG_ITMP3);
4493
4494
4495         /* process patcher calls **************************************************/
4496
4497         {
4498                 u1          *xcodeptr;
4499                 patchref    *pref;
4500                 codegendata *tmpcd;
4501                 ptrint       mcode;
4502
4503                 tmpcd = DNEW(codegendata);
4504
4505                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4506                         /* Get machine code which is patched back in later. A             */
4507                         /* `call rel32' is 5 bytes long (but read 8 bytes).               */
4508
4509                         xcodeptr = cd->mcodebase + pref->branchpos;
4510                         mcode = *((ptrint *) xcodeptr);
4511
4512                         /* patch in `call rel32' to call the following code               */
4513
4514                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
4515                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
4516
4517                         /* move pointer to java_objectheader onto stack */
4518
4519 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4520                         /* create a virtual java_objectheader */
4521
4522                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4523                         off = dseg_addaddress(cd, NULL);                    /* vftbl      */
4524
4525                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + off, REG_ITMP3);
4526                         x86_64_push_reg(cd, REG_ITMP3);
4527 #else
4528                         x86_64_push_imm(cd, 0);
4529 #endif
4530
4531                         /* move machine code bytes and classinfo pointer into registers */
4532
4533                         x86_64_mov_imm_reg(cd, (ptrint) mcode, REG_ITMP3);
4534                         x86_64_push_reg(cd, REG_ITMP3);
4535                         x86_64_mov_imm_reg(cd, (ptrint) pref->ref, REG_ITMP3);
4536                         x86_64_push_reg(cd, REG_ITMP3);
4537
4538                         x86_64_mov_imm_reg(cd, (ptrint) pref->patcher, REG_ITMP3);
4539                         x86_64_push_reg(cd, REG_ITMP3);
4540
4541                         x86_64_mov_imm_reg(cd, (ptrint) asm_wrapper_patcher, REG_ITMP3);
4542                         x86_64_jmp_reg(cd, REG_ITMP3);
4543                 }
4544         }
4545
4546         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
4547
4548         return m->entrypoint;
4549 }
4550
4551
4552 /*
4553  * These are local overrides for various environment variables in Emacs.
4554  * Please do not remove this and leave it at the end of the file, where
4555  * Emacs will automagically detect them.
4556  * ---------------------------------------------------------------------
4557  * Local variables:
4558  * mode: c
4559  * indent-tabs-mode: t
4560  * c-basic-offset: 4
4561  * tab-width: 4
4562  * End:
4563  */