src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31             Edwin Steiner
  32
  33    $Id: codegen.c 5339 2006-09-05 21:14:00Z edwin $
  34
  35 */
  36
  37
  38 #include "config.h"
  39
  40 #include <assert.h>
  41 #include <stdio.h>
  42
  43 #include "vm/types.h"
  44
  45 #include "md.h"
  46 #include "md-abi.h"
  47
  48 #include "vm/jit/x86_64/arch.h"
  49 #include "vm/jit/x86_64/codegen.h"
  50 #include "vm/jit/x86_64/md-emit.h"
  51
  52 #include "mm/memory.h"
  53 #include "native/jni.h"
  54 #include "native/native.h"
  55
  56 #if defined(ENABLE_THREADS)
  57 # include "threads/native/lock.h"
  58 #endif
  59
  60 #include "vm/builtin.h"
  61 #include "vm/exceptions.h"
  62 #include "vm/global.h"
  63 #include "vm/loader.h"
  64 #include "vm/options.h"
  65 #include "vm/statistics.h"
  66 #include "vm/stringlocal.h"
  67 #include "vm/vm.h"
  68 #include "vm/jit/asmpart.h"
  69 #include "vm/jit/codegen-common.h"
  70 #include "vm/jit/dseg.h"
  71 #include "vm/jit/emit.h"
  72 #include "vm/jit/jit.h"
  73 #include "vm/jit/methodheader.h"
  74 #include "vm/jit/parse.h"
  75 #include "vm/jit/patcher.h"
  76 #include "vm/jit/reg.h"
  77 #include "vm/jit/replace.h"
  78
  79 #if defined(ENABLE_LSRA)
  80 # include "vm/jit/allocator/lsra.h"
  81 #endif
  82
  83
  84 /* codegen *********************************************************************
  85
  86    Generates machine code.
  87
  88 *******************************************************************************/
  89
  90 bool codegen(jitdata *jd)
  91 {
  92         methodinfo         *m;
  93         codeinfo           *code;
  94         codegendata        *cd;
  95         registerdata       *rd;
  96         s4                  len, s1, s2, s3, d, disp;
  97         u2                  currentline;
  98         ptrint              a;
  99         s4                  fieldtype;
 100         stackptr            src;
 101         varinfo            *var;
 102         basicblock         *bptr;
 103         new_instruction    *iptr;
 104         exceptiontable     *ex;
 105         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
 106         builtintable_entry *bte;
 107         methoddesc         *md;
 108         rplpoint           *replacementpoint;
 109
 110         /* get required compiler data */
 111
 112         m    = jd->m;
 113         code = jd->code;
 114         cd   = jd->cd;
 115         rd   = jd->rd;
 116
 117         /* prevent compiler warnings */
 118
 119         d = 0;
 120         lm = NULL;
 121         bte = NULL;
 122
 123         {
 124         s4 i, p, t, l;
 125         s4 savedregs_num;
 126
 127         savedregs_num = 0;
 128
 129         /* space to save used callee saved registers */
 130
 131         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 132         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 133
 134         cd->stackframesize = rd->memuse + savedregs_num;
 135
 136 #if defined(ENABLE_THREADS)
 137         /* space to save argument of monitor_enter */
 138
 139         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 140                 cd->stackframesize++;
 141 #endif
 142
 143         /* Keep stack of non-leaf functions 16-byte aligned for calls into
 144            native code e.g. libc or jni (alignment problems with
 145            movaps). */
 146
 147         if (!jd->isleafmethod || opt_verbosecall)
 148                 cd->stackframesize |= 0x1;
 149
 150         /* create method header */
 151
 152         (void) dseg_addaddress(cd, code);                      /* CodeinfoPointer */
 153         (void) dseg_adds4(cd, cd->stackframesize * 8);         /* FrameSize       */
 154
 155 #if defined(ENABLE_THREADS)
 156         /* IsSync contains the offset relative to the stack pointer for the
 157            argument of monitor_exit used in the exception handler. Since the
 158            offset could be zero and give a wrong meaning of the flag it is
 159            offset by one.
 160         */
 161
 162         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 163                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);       /* IsSync          */
 164         else
 165 #endif
 166                 (void) dseg_adds4(cd, 0);                          /* IsSync          */
 167
 168         (void) dseg_adds4(cd, jd->isleafmethod);               /* IsLeaf          */
 169         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse); /* IntSave         */
 170         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse); /* FltSave         */
 171
 172         (void) dseg_addlinenumbertablesize(cd);
 173
 174         (void) dseg_adds4(cd, cd->exceptiontablelength);       /* ExTableSize     */
 175
 176         /* create exception table */
 177
 178         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 179                 dseg_addtarget(cd, ex->start);
 180                 dseg_addtarget(cd, ex->end);
 181                 dseg_addtarget(cd, ex->handler);
 182                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 183         }
 184
 185         /* generate method profiling code */
 186
 187         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 188                 /* count frequency */
 189
 190                 M_MOV_IMM(code, REG_ITMP3);
 191                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
 192
 193                 PROFILE_CYCLE_START;
 194         }
 195
 196         /* create stack frame (if necessary) */
 197
 198         if (cd->stackframesize)
 199                 M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
 200
 201         /* save used callee saved registers */
 202
 203         p = cd->stackframesize;
 204         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 205                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 206         }
 207         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 208                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 209         }
 210
 211         /* take arguments out of register or stack frame */
 212
 213         md = m->parseddesc;
 214
 215         for (p = 0, l = 0; p < md->paramcount; p++) {
 216                 t = md->paramtypes[p].type;
 217                 var = &(rd->locals[l][t]);
 218                 l++;
 219                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 220                         l++;
 221                 if (var->type < 0)
 222                         continue;
 223                 s1 = md->params[p].regoff;
 224                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 225                         s2 = rd->argintregs[s1];
 226                         if (!md->params[p].inmemory) {           /* register arguments    */
 227                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 228                                         M_INTMOVE(s2, var->regoff);
 229
 230                                 } else {                             /* reg arg -> spilled    */
 231                                     M_LST(s2, REG_SP, var->regoff * 8);
 232                                 }
 233
 234                         } else {                                 /* stack arguments       */
 235                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 236                                         /* + 8 for return address */
 237                                         M_LLD(var->regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 238
 239                                 } else {                             /* stack arg -> spilled  */
 240                                         var->regoff = cd->stackframesize + s1 + 1;
 241                                 }
 242                         }
 243
 244                 } else {                                     /* floating args         */
 245                         if (!md->params[p].inmemory) {           /* register arguments    */
 246                                 s2 = rd->argfltregs[s1];
 247                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 248                                         M_FLTMOVE(s2, var->regoff);
 249
 250                                 } else {                                         /* reg arg -> spilled    */
 251                                         M_DST(s2, REG_SP, var->regoff * 8);
 252                                 }
 253
 254                         } else {                                 /* stack arguments       */
 255                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 256                                         M_DLD(var->regoff, REG_SP, (cd->stackframesize + s1) * 8 + 8);
 257
 258                                 } else {
 259                                         var->regoff = cd->stackframesize + s1 + 1;
 260                                 }
 261                         }
 262                 }
 263         }  /* end for */
 264
 265         /* save monitorenter argument */
 266
 267 #if defined(ENABLE_THREADS)
 268         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 269                 /* stack offset for monitor argument */
 270
 271                 s1 = rd->memuse;
 272
 273                 if (opt_verbosecall) {
 274                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 275
 276                         for (p = 0; p < INT_ARG_CNT; p++)
 277                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 278
 279                         for (p = 0; p < FLT_ARG_CNT; p++)
 280                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 281
 282                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 283                 }
 284
 285                 /* decide which monitor enter function to call */
 286
 287                 if (m->flags & ACC_STATIC) {
 288                         M_MOV_IMM(&m->class->object.header, rd->argintregs[0]);
 289                 }
 290                 else {
 291                         M_TEST(rd->argintregs[0]);
 292                         M_BEQ(0);
 293                         codegen_add_nullpointerexception_ref(cd);
 294                 }
 295
 296                 M_AST(rd->argintregs[0], REG_SP, s1 * 8);
 297                 M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
 298                 M_CALL(REG_ITMP1);
 299
 300                 if (opt_verbosecall) {
 301                         for (p = 0; p < INT_ARG_CNT; p++)
 302                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 303
 304                         for (p = 0; p < FLT_ARG_CNT; p++)
 305                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 306
 307                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 308                 }
 309         }
 310 #endif
 311
 312 #if !defined(NDEBUG)
 313         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
 314                 emit_verbosecall_enter(jd);
 315 #endif /* !defined(NDEBUG) */
 316
 317         }
 318
 319         /* end of header generation */
 320
 321         replacementpoint = jd->code->rplpoints;
 322
 323         /* walk through all basic blocks */
 324
 325         for (bptr = jd->new_basicblocks; bptr != NULL; bptr = bptr->next) {
 326
 327                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 328
 329                 if (bptr->flags >= BBREACHED) {
 330
 331                         /* branch resolving */
 332
 333                         branchref *bref;
 334                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 335                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 336                                                                   bref->branchpos,
 337                                                                   bptr->mpc);
 338                         }
 339
 340                 /* handle replacement points */
 341
 342 #if 0
 343                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 344                         replacementpoint->pc = (u1*)(ptrint)bptr->mpc; /* will be resolved later */
 345
 346                         replacementpoint++;
 347
 348                         assert(cd->lastmcodeptr <= cd->mcodeptr);
 349                         cd->lastmcodeptr = cd->mcodeptr + 5; /* 5 byte jmp patch */
 350                 }
 351 #endif
 352
 353                 /* copy interface registers to their destination */
 354
 355                 len = bptr->indepth;
 356                 MCODECHECK(512);
 357
 358                 /* generate basicblock profiling code */
 359
 360                 if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
 361                         /* count frequency */
 362
 363                         M_MOV_IMM(code->bbfrequency, REG_ITMP3);
 364                         M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
 365
 366                         /* if this is an exception handler, start profiling again */
 367
 368                         if (bptr->type == BBTYPE_EXH)
 369                                 PROFILE_CYCLE_START;
 370                 }
 371
 372 #if defined(ENABLE_LSRA)
 373                 if (opt_lsra) {
 374                         while (len) {
 375                                 len--;
 376                                 src = bptr->invars[len];
 377                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 378                                         if (bptr->type == BBTYPE_SBR) {
 379 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 380                                                 if (!(src->flags & INMEMORY))
 381                                                         d = src->regoff;
 382                                                 else
 383                                                         d = REG_ITMP1;
 384                                                 M_POP(d);
 385                                                 emit_store(jd, NULL, src, d);
 386
 387                                         } else if (bptr->type == BBTYPE_EXH) {
 388 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 389                                                 if (!(src->flags & INMEMORY))
 390                                                         d= src->regoff;
 391                                                 else
 392                                                         d=REG_ITMP1;
 393                                                 M_INTMOVE(REG_ITMP1, d);
 394                                                 emit_store(jd, NULL, src, d);
 395                                         }
 396                                 }
 397                         }
 398
 399                 } else {
 400 #endif
 401
 402                 while (len) {
 403                         len--;
 404                         src = bptr->invars[len];
 405                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 406                                 if (bptr->type == BBTYPE_SBR) {
 407                                         d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 408                                         M_POP(d);
 409                                         emit_store(jd, NULL, src, d);
 410
 411                                 } else if (bptr->type == BBTYPE_EXH) {
 412                                         d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 413                                         M_INTMOVE(REG_ITMP1, d);
 414                                         emit_store(jd, NULL, src, d);
 415                                 }
 416
 417                         } else {
 418                                 d = codegen_reg_of_var(rd, 0, src, REG_ITMP1);
 419                                 if ((src->varkind != STACKVAR)) {
 420                                         s2 = src->type;
 421                                         if (IS_FLT_DBL_TYPE(s2)) {
 422                                                 s1 = rd->interfaces[len][s2].regoff;
 423
 424                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 425                                                         M_FLTMOVE(s1, d);
 426                                                 else
 427                                                         M_DLD(d, REG_SP, s1 * 8);
 428
 429                                                 emit_store(jd, NULL, src, d);
 430
 431                                         } else {
 432                                                 s1 = rd->interfaces[len][s2].regoff;
 433
 434                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 435                                                         M_INTMOVE(s1, d);
 436                                                 else
 437                                                         M_LLD(d, REG_SP, s1 * 8);
 438
 439                                                 emit_store(jd, NULL, src, d);
 440                                         }
 441                                 }
 442                         }
 443                 }
 444 #if defined(ENABLE_LSRA)
 445                 }
 446 #endif
 447                 /* walk through all instructions */
 448
 449                 len = bptr->icount;
 450                 currentline = 0;
 451
 452                 for (iptr = bptr->iinstr; len > 0; len--, iptr++) {
 453                         if (iptr->line != currentline) {
 454                                 dseg_addlinenumber(cd, iptr->line);
 455                                 currentline = iptr->line;
 456                         }
 457
 458                         MCODECHECK(1024);                         /* 1KB should be enough */
 459
 460                         switch (iptr->opc) {
 461                         case ICMD_INLINE_START: /* internal ICMDs                         */
 462                         case ICMD_INLINE_END:
 463                                 break;
 464
 465                         case ICMD_NOP:    /* ...  ==> ...                                 */
 466                                 break;
 467
 468                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 469
 470                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 471                                 M_TEST(s1);
 472                                 M_BEQ(0);
 473                                 codegen_add_nullpointerexception_ref(cd);
 474                                 break;
 475
 476                 /* constant operations ************************************************/
 477
 478                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 479
 480                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 481                         ICONST(d, iptr->sx.val.i);
 482                         emit_store_dst(jd, iptr, d);
 483                         break;
 484
 485                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 486
 487                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 488                         LCONST(d, iptr->sx.val.l);
 489                         emit_store_dst(jd, iptr, d);
 490                         break;
 491
 492                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 493
 494                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 495                         disp = dseg_addfloat(cd, iptr->sx.val.f);
 496                         emit_movdl_membase_reg(cd, RIP, -((cd->mcodeptr + ((d > 7) ? 9 : 8)) - cd->mcodebase) + disp, d);
 497                         emit_store_dst(jd, iptr, d);
 498                         break;
 499
 500                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 501
 502                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 503                         disp = dseg_adddouble(cd, iptr->sx.val.d);
 504                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, d);
 505                         emit_store_dst(jd, iptr, d);
 506                         break;
 507
 508                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 509
 510                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 511
 512                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
 513 /*                              PROFILE_CYCLE_STOP; */
 514
 515                                 codegen_addpatchref(cd, PATCHER_aconst,
 516                                                                         iptr->sx.val.c.ref, 0);
 517
 518                                 if (opt_showdisassemble) {
 519                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
 520                                 }
 521
 522 /*                              PROFILE_CYCLE_START; */
 523
 524                                 M_MOV_IMM(NULL, d);
 525
 526                         } else {
 527                                 if (iptr->sx.val.anyptr == 0)
 528                                         M_CLR(d);
 529                                 else
 530                                         M_MOV_IMM(iptr->sx.val.anyptr, d);
 531                         }
 532                         emit_store_dst(jd, iptr, d);
 533                         break;
 534
 535
 536                 /* load/store operations **********************************************/
 537
 538                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 539                                       /* s1.localindex = local variable                         */
 540
 541                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 542                         if ((iptr->dst.var->varkind == LOCALVAR) &&
 543                                 (iptr->dst.var->varnum == iptr->s1.localindex))
 544                                 break;
 545                         var = &(rd->locals[iptr->s1.localindex][iptr->opc - ICMD_ILOAD]);
 546                         if (var->flags & INMEMORY)
 547                                 M_ILD(d, REG_SP, var->regoff * 8);
 548                         else
 549                                 M_INTMOVE(var->regoff, d);
 550                         emit_store_dst(jd, iptr, d);
 551                         break;
 552
 553                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 554                 case ICMD_ALOAD:      /* op1 = local variable                         */
 555
 556                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 557                         if ((iptr->dst.var->varkind == LOCALVAR) &&
 558                                 (iptr->dst.var->varnum == iptr->s1.localindex))
 559                                 break;
 560                         var = &(rd->locals[iptr->s1.localindex][iptr->opc - ICMD_ILOAD]);
 561                         if (var->flags & INMEMORY)
 562                                 M_LLD(d, REG_SP, var->regoff * 8);
 563                         else
 564                                 M_INTMOVE(var->regoff, d);
 565                         emit_store_dst(jd, iptr, d);
 566                         break;
 567
 568                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 569                                       /* s1.localindex = local variable                         */
 570
 571                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 572                         if ((iptr->dst.var->varkind == LOCALVAR) &&
 573                             (iptr->dst.var->varnum == iptr->s1.localindex))
 574                                 break;
 575                         var = &(rd->locals[iptr->s1.localindex][iptr->opc - ICMD_ILOAD]);
 576                         if (var->flags & INMEMORY)
 577                                 M_FLD(d, REG_SP, var->regoff * 8);
 578                         else
 579                                 M_FLTMOVE(var->regoff, d);
 580                         emit_store_dst(jd, iptr, d);
 581                         break;
 582
 583                 case ICMD_DLOAD:      /* ...  ==> ..., content of local variable      */
 584                                       /* s1.localindex = local variable                         */
 585
 586                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
 587                         if ((iptr->dst.var->varkind == LOCALVAR) &&
 588                             (iptr->dst.var->varnum == iptr->s1.localindex))
 589                                 break;
 590                         var = &(rd->locals[iptr->s1.localindex][iptr->opc - ICMD_ILOAD]);
 591                         if (var->flags & INMEMORY)
 592                                 M_DLD(d, REG_SP, var->regoff * 8);
 593                         else
 594                                 M_FLTMOVE(var->regoff, d);
 595                         emit_store_dst(jd, iptr, d);
 596                         break;
 597
 598                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 599                                       /* dst.localindex = local variable                         */
 600
 601                         if ((iptr->s1.var->varkind == LOCALVAR) && (iptr->s1.var->varnum == iptr->dst.localindex))
 602                                 break;
 603                         var = &(rd->locals[iptr->dst.localindex][iptr->opc - ICMD_ISTORE]);
 604                         if (var->flags & INMEMORY) {
 605                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 606                                 M_IST(s1, REG_SP, var->regoff * 8);
 607                         } else {
 608                                 s1 = emit_load_s1(jd, iptr, var->regoff);
 609                                 M_INTMOVE(s1, var->regoff);
 610                         }
 611                         break;
 612
 613                 case ICMD_LSTORE:     /* ..., value  ==> ...                          */
 614                 case ICMD_ASTORE:     /* op1 = local variable                         */
 615
 616                         if ((iptr->s1.var->varkind == LOCALVAR) && (iptr->s1.var->varnum == iptr->dst.localindex))
 617                                 break;
 618                         var = &(rd->locals[iptr->dst.localindex][iptr->opc - ICMD_ISTORE]);
 619                         if (var->flags & INMEMORY) {
 620                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 621                                 M_LST(s1, REG_SP, var->regoff * 8);
 622                         } else {
 623                                 s1 = emit_load_s1(jd, iptr, var->regoff);
 624                                 M_INTMOVE(s1, var->regoff);
 625                         }
 626                         break;
 627
 628                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 629                                       /* dst.localindex = local variable                         */
 630
 631                         if ((iptr->s1.var->varkind == LOCALVAR) &&
 632                             (iptr->s1.var->varnum == iptr->dst.localindex)) {
 633                                 break;
 634                         }
 635                         var = &(rd->locals[iptr->dst.localindex][iptr->opc - ICMD_ISTORE]);
 636                         if (var->flags & INMEMORY) {
 637                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 638                                 M_FST(s1, REG_SP, var->regoff * 8);
 639                         } else {
 640                                 s1 = emit_load_s1(jd, iptr, var->regoff);
 641                                 M_FLTMOVE(s1, var->regoff);
 642                         }
 643                         break;
 644
 645                 case ICMD_DSTORE:     /* ..., value  ==> ...                          */
 646                                       /* dst.localindex = local variable                         */
 647
 648                         if ((iptr->s1.var->varkind == LOCALVAR) &&
 649                             (iptr->s1.var->varnum == iptr->dst.localindex)) {
 650                                 break;
 651                         }
 652                         var = &(rd->locals[iptr->dst.localindex][iptr->opc - ICMD_ISTORE]);
 653                         if (var->flags & INMEMORY) {
 654                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
 655                                 M_DST(s1, REG_SP, var->regoff * 8);
 656                         } else {
 657                                 s1 = emit_load_s1(jd, iptr, var->regoff);
 658                                 M_FLTMOVE(s1, var->regoff);
 659                         }
 660                         break;
 661
 662
 663                 /* pop/dup/swap operations ********************************************/
 664
 665                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 666
 667                 case ICMD_POP:        /* ..., value  ==> ...                          */
 668                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 669                         break;
 670
 671                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 672
 673                         M_COPY(iptr->s1.var, iptr->dst.var);
 674                         break;
 675
 676                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 677
 678                         M_COPY(iptr->dst.dupslots[  1], iptr->dst.dupslots[2+2]);
 679                         M_COPY(iptr->dst.dupslots[  0], iptr->dst.dupslots[2+1]);
 680                         M_COPY(iptr->dst.dupslots[2+2], iptr->dst.dupslots[2+0]);
 681                         break;
 682
 683                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 684
 685                         M_COPY(iptr->dst.dupslots[  2], iptr->dst.dupslots[3+3]);
 686                         M_COPY(iptr->dst.dupslots[  1], iptr->dst.dupslots[3+2]);
 687                         M_COPY(iptr->dst.dupslots[  0], iptr->dst.dupslots[3+1]);
 688                         M_COPY(iptr->dst.dupslots[3+3], iptr->dst.dupslots[3+0]);
 689                         break;
 690
 691                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 692
 693                         M_COPY(iptr->dst.dupslots[  1], iptr->dst.dupslots[2+1]);
 694                         M_COPY(iptr->dst.dupslots[  0], iptr->dst.dupslots[2+0]);
 695                         break;
 696
 697                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 698
 699                         M_COPY(iptr->dst.dupslots[  2], iptr->dst.dupslots[3+4]);
 700                         M_COPY(iptr->dst.dupslots[  1], iptr->dst.dupslots[3+3]);
 701                         M_COPY(iptr->dst.dupslots[  0], iptr->dst.dupslots[3+2]);
 702                         M_COPY(iptr->dst.dupslots[3+4], iptr->dst.dupslots[3+1]);
 703                         M_COPY(iptr->dst.dupslots[3+3], iptr->dst.dupslots[3+0]);
 704                         break;
 705
 706                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 707
 708                         M_COPY(iptr->dst.dupslots[  3], iptr->dst.dupslots[4+5]);
 709                         M_COPY(iptr->dst.dupslots[  2], iptr->dst.dupslots[4+4]);
 710                         M_COPY(iptr->dst.dupslots[  1], iptr->dst.dupslots[4+3]);
 711                         M_COPY(iptr->dst.dupslots[  0], iptr->dst.dupslots[4+2]);
 712                         M_COPY(iptr->dst.dupslots[4+5], iptr->dst.dupslots[4+1]);
 713                         M_COPY(iptr->dst.dupslots[4+4], iptr->dst.dupslots[4+0]);
 714                         break;
 715
 716                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 717
 718                         M_COPY(iptr->dst.dupslots[  1], iptr->dst.dupslots[2+0]);
 719                         M_COPY(iptr->dst.dupslots[  0], iptr->dst.dupslots[2+1]);
 720                         break;
 721
 722
 723                 /* integer operations *************************************************/
 724
 725                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 726
 727                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 728                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 729                         M_INTMOVE(s1, d);
 730                         M_INEG(d);
 731                         emit_store_dst(jd, iptr, d);
 732                         break;
 733
 734                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 735
 736                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 737                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 738                         M_INTMOVE(s1, d);
 739                         M_LNEG(d);
 740                         emit_store_dst(jd, iptr, d);
 741                         break;
 742
 743                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 744
 745                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 746                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 747                         M_ISEXT(s1, d);
 748                         emit_store_dst(jd, iptr, d);
 749                         break;
 750
 751                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 752
 753                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 754                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 755                         M_IMOV(s1, d);
 756                         emit_store_dst(jd, iptr, d);
 757                         break;
 758
 759                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 760
 761                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 762                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 763                         M_BSEXT(s1, d);
 764                         emit_store_dst(jd, iptr, d);
 765                         break;
 766
 767                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 768
 769                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 770                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 771                         M_CZEXT(s1, d);
 772                         emit_store_dst(jd, iptr, d);
 773                         break;
 774
 775                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 776
 777                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 778                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
 779                         M_SSEXT(s1, d);
 780                         emit_store_dst(jd, iptr, d);
 781                         break;
 782
 783
 784                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 785
 786                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 787                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 788                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 789                         if (s2 == d)
 790                                 M_IADD(s1, d);
 791                         else {
 792                                 M_INTMOVE(s1, d);
 793                                 M_IADD(s2, d);
 794                         }
 795                         emit_store_dst(jd, iptr, d);
 796                         break;
 797
 798                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 799                                       /* sx.val.i = constant                             */
 800
 801                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 802                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 803                         M_INTMOVE(s1, d);
 804                         M_IADD_IMM(iptr->sx.val.i, d);
 805                         emit_store_dst(jd, iptr, d);
 806                         break;
 807
 808                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 809
 810                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 811                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 812                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 813                         if (s2 == d)
 814                                 M_LADD(s1, d);
 815                         else {
 816                                 M_INTMOVE(s1, d);
 817                                 M_LADD(s2, d);
 818                         }
 819                         emit_store_dst(jd, iptr, d);
 820                         break;
 821
 822                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 823                                       /* sx.val.l = constant                             */
 824
 825                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 826                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 827                         M_INTMOVE(s1, d);
 828                         if (IS_IMM32(iptr->sx.val.l))
 829                                 M_LADD_IMM(iptr->sx.val.l, d);
 830                         else {
 831                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 832                                 M_LADD(REG_ITMP2, d);
 833                         }
 834                         emit_store_dst(jd, iptr, d);
 835                         break;
 836
 837                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 838
 839                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 840                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 841                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 842                         if (s2 == d) {
 843                                 M_INTMOVE(s1, REG_ITMP1);
 844                                 M_ISUB(s2, REG_ITMP1);
 845                                 M_INTMOVE(REG_ITMP1, d);
 846                         } else {
 847                                 M_INTMOVE(s1, d);
 848                                 M_ISUB(s2, d);
 849                         }
 850                         emit_store_dst(jd, iptr, d);
 851                         break;
 852
 853                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 854                                       /* sx.val.i = constant                             */
 855
 856                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 857                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 858                         M_INTMOVE(s1, d);
 859                         M_ISUB_IMM(iptr->sx.val.i, d);
 860                         emit_store_dst(jd, iptr, d);
 861                         break;
 862
 863                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 864
 865                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 866                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 867                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 868                         if (s2 == d) {
 869                                 M_INTMOVE(s1, REG_ITMP1);
 870                                 M_LSUB(s2, REG_ITMP1);
 871                                 M_INTMOVE(REG_ITMP1, d);
 872                         } else {
 873                                 M_INTMOVE(s1, d);
 874                                 M_LSUB(s2, d);
 875                         }
 876                         emit_store_dst(jd, iptr, d);
 877                         break;
 878
 879                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
 880                                       /* sx.val.l = constant                             */
 881
 882                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 883                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 884                         M_INTMOVE(s1, d);
 885                         if (IS_IMM32(iptr->sx.val.l))
 886                                 M_LSUB_IMM(iptr->sx.val.l, d);
 887                         else {
 888                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 889                                 M_LSUB(REG_ITMP2, d);
 890                         }
 891                         emit_store_dst(jd, iptr, d);
 892                         break;
 893
 894                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 895
 896                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 897                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 898                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 899                         if (s2 == d)
 900                                 M_IMUL(s1, d);
 901                         else {
 902                                 M_INTMOVE(s1, d);
 903                                 M_IMUL(s2, d);
 904                         }
 905                         emit_store_dst(jd, iptr, d);
 906                         break;
 907
 908                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
 909                                       /* sx.val.i = constant                             */
 910
 911                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 912                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 913                         if (iptr->sx.val.i == 2) {
 914                                 M_INTMOVE(s1, d);
 915                                 M_ISLL_IMM(1, d);
 916                         } else
 917                                 M_IMUL_IMM(s1, iptr->sx.val.i, d);
 918                         emit_store_dst(jd, iptr, d);
 919                         break;
 920
 921                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
 922
 923                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 924                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
 925                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
 926                         if (s2 == d)
 927                                 M_LMUL(s1, d);
 928                         else {
 929                                 M_INTMOVE(s1, d);
 930                                 M_LMUL(s2, d);
 931                         }
 932                         emit_store_dst(jd, iptr, d);
 933                         break;
 934
 935                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
 936                                       /* sx.val.l = constant                             */
 937
 938                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
 939                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
 940                         if (IS_IMM32(iptr->sx.val.l))
 941                                 M_LMUL_IMM(s1, iptr->sx.val.l, d);
 942                         else {
 943                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
 944                                 M_INTMOVE(s1, d);
 945                                 M_LMUL(REG_ITMP2, d);
 946                         }
 947                         emit_store_dst(jd, iptr, d);
 948                         break;
 949
 950                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
 951
 952                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 953                 if (iptr->s1.var->flags & INMEMORY)
 954                                 M_ILD(RAX, REG_SP, iptr->s1.var->regoff * 8);
 955                         else
 956                                 M_INTMOVE(iptr->s1.var->regoff, RAX);
 957
 958                         if (iptr->sx.s23.s2.var->flags & INMEMORY)
 959                                 M_ILD(REG_ITMP3, REG_SP, iptr->sx.s23.s2.var->regoff * 8);
 960                         else
 961                                 M_INTMOVE(iptr->sx.s23.s2.var->regoff, REG_ITMP3);
 962
 963                         if (checknull) {
 964                                 M_ITEST(REG_ITMP3);
 965                                 M_BEQ(0);
 966                                 codegen_add_arithmeticexception_ref(cd);
 967                         }
 968
 969                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
 970                         emit_jcc(cd, CC_NE, 4 + 6);
 971                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
 972                         emit_jcc(cd, CC_E, 3 + 1 + 3);                /* 6 bytes */
 973
 974                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
 975                         emit_cltd(cd);
 976                         emit_idivl_reg(cd, REG_ITMP3);
 977
 978                         if (iptr->dst.var->flags & INMEMORY) {
 979                                 emit_mov_reg_membase(cd, RAX, REG_SP, iptr->dst.var->regoff * 8);
 980                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
 981
 982                         } else {
 983                                 M_INTMOVE(RAX, iptr->dst.var->regoff);
 984
 985                                 if (iptr->dst.var->regoff != RDX) {
 986                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
 987                                 }
 988                         }
 989                         break;
 990
 991                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
 992
 993                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
 994                         if (iptr->s1.var->flags & INMEMORY)
 995                                 M_ILD(RAX, REG_SP, iptr->s1.var->regoff * 8);
 996                         else
 997                                 M_INTMOVE(iptr->s1.var->regoff, RAX);
 998
 999                         if (iptr->sx.s23.s2.var->flags & INMEMORY)
1000                                 M_ILD(REG_ITMP3, REG_SP, iptr->sx.s23.s2.var->regoff * 8);
1001                         else
1002                                 M_INTMOVE(iptr->sx.s23.s2.var->regoff, REG_ITMP3);
1003
1004                         if (checknull) {
1005                                 M_ITEST(REG_ITMP3);
1006                                 M_BEQ(0);
1007                                 codegen_add_arithmeticexception_ref(cd);
1008                         }
1009
1010                         emit_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1011
1012                         emit_alul_imm_reg(cd, ALU_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1013                         emit_jcc(cd, CC_NE, 2 + 4 + 6);
1014
1015
1016                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
1017                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP3);    /* 4 bytes */
1018                         emit_jcc(cd, CC_E, 1 + 3);                    /* 6 bytes */
1019
1020                         emit_cltd(cd);
1021                         emit_idivl_reg(cd, REG_ITMP3);
1022
1023                         if (iptr->dst.var->flags & INMEMORY) {
1024                                 emit_mov_reg_membase(cd, RDX, REG_SP, iptr->dst.var->regoff * 8);
1025                                 emit_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1026
1027                         } else {
1028                                 M_INTMOVE(RDX, iptr->dst.var->regoff);
1029
1030                                 if (iptr->dst.var->regoff != RDX) {
1031                                         emit_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1032                                 }
1033                         }
1034                         break;
1035
1036                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1037                                       /* sx.val.i = constant                             */
1038
1039                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1040                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1041                         M_INTMOVE(s1, REG_ITMP1);
1042                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1043                         emit_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
1044                         emit_cmovccl_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
1045                         emit_shiftl_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
1046                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1047                         emit_store_dst(jd, iptr, d);
1048                         break;
1049
1050                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1051                                       /* sx.val.i = constant                             */
1052
1053                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1054                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1055                         M_INTMOVE(s1, REG_ITMP1);
1056                         emit_alul_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1057                         emit_leal_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
1058                         emit_cmovccl_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
1059                         emit_alul_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
1060                         emit_alul_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
1061                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1062                         emit_store_dst(jd, iptr, d);
1063                         break;
1064
1065
1066                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1067
1068                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1069
1070                 if (iptr->s1.var->flags & INMEMORY)
1071                                 M_LLD(RAX, REG_SP, iptr->s1.var->regoff * 8);
1072                         else
1073                                 M_INTMOVE(iptr->s1.var->regoff, RAX);
1074
1075                         if (iptr->sx.s23.s2.var->flags & INMEMORY)
1076                                 M_LLD(REG_ITMP3, REG_SP, iptr->sx.s23.s2.var->regoff * 8);
1077                         else
1078                                 M_INTMOVE(iptr->sx.s23.s2.var->regoff, REG_ITMP3);
1079
1080                         if (checknull) {
1081                                 M_TEST(REG_ITMP3);
1082                                 M_BEQ(0);
1083                                 codegen_add_arithmeticexception_ref(cd);
1084                         }
1085
1086                         /* check as described in jvm spec */
1087                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1088                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, RAX);
1089                         M_BNE(4 + 6);
1090                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1091                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
1092
1093                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1094                         emit_cqto(cd);
1095                         emit_idiv_reg(cd, REG_ITMP3);
1096
1097                         if (iptr->dst.var->flags & INMEMORY) {
1098                                 M_LST(RAX, REG_SP, iptr->dst.var->regoff * 8);
1099                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1100
1101                         } else {
1102                                 M_INTMOVE(RAX, iptr->dst.var->regoff);
1103
1104                                 if (iptr->dst.var->regoff != RDX) {
1105                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1106                                 }
1107                         }
1108                         break;
1109
1110                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1111
1112                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1113                         if (iptr->s1.var->flags & INMEMORY)
1114                                 M_LLD(REG_ITMP1, REG_SP, iptr->s1.var->regoff * 8);
1115                         else
1116                                 M_INTMOVE(iptr->s1.var->regoff, REG_ITMP1);
1117
1118                         if (iptr->sx.s23.s2.var->flags & INMEMORY)
1119                                 M_LLD(REG_ITMP3, REG_SP, iptr->sx.s23.s2.var->regoff * 8);
1120                         else
1121                                 M_INTMOVE(iptr->sx.s23.s2.var->regoff, REG_ITMP3);
1122
1123                         if (checknull) {
1124                                 M_ITEST(REG_ITMP3);
1125                                 M_BEQ(0);
1126                                 codegen_add_arithmeticexception_ref(cd);
1127                         }
1128
1129                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1130
1131                         /* check as described in jvm spec */
1132                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1133                         M_LCMP_MEMBASE(RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP1);
1134                         M_BNE(3 + 4 + 6);
1135
1136 #if 0
1137                         emit_alul_reg_reg(cd, ALU_XOR, RDX, RDX);         /* 2 bytes */
1138 #endif
1139                         M_LXOR(RDX, RDX);                                      /* 3 bytes */
1140                         M_LCMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1141                         M_BEQ(2 + 3);                                          /* 6 bytes */
1142
1143                         emit_cqto(cd);
1144                         emit_idiv_reg(cd, REG_ITMP3);
1145
1146                         if (iptr->dst.var->flags & INMEMORY) {
1147                                 M_LST(RDX, REG_SP, iptr->dst.var->regoff * 8);
1148                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1149
1150                         } else {
1151                                 M_INTMOVE(RDX, iptr->dst.var->regoff);
1152
1153                                 if (iptr->dst.var->regoff != RDX) {
1154                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1155                                 }
1156                         }
1157                         break;
1158
1159                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1160                                       /* sx.val.i = constant                             */
1161
1162                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1163                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1164                         M_INTMOVE(s1, REG_ITMP1);
1165                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1166                         emit_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->sx.val.i) - 1, REG_ITMP2);
1167                         emit_cmovcc_reg_reg(cd, CC_LE, REG_ITMP2, REG_ITMP1);
1168                         emit_shift_imm_reg(cd, SHIFT_SAR, iptr->sx.val.i, REG_ITMP1);
1169                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1170                         emit_store_dst(jd, iptr, d);
1171                         break;
1172
1173                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1174                                       /* sx.val.l = constant                             */
1175
1176                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1177                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1178                         M_INTMOVE(s1, REG_ITMP1);
1179                         emit_alu_imm_reg(cd, ALU_CMP, -1, REG_ITMP1);
1180                         emit_lea_membase_reg(cd, REG_ITMP1, iptr->sx.val.i, REG_ITMP2);
1181                         emit_cmovcc_reg_reg(cd, CC_G, REG_ITMP1, REG_ITMP2);
1182                         emit_alu_imm_reg(cd, ALU_AND, -1 - (iptr->sx.val.i), REG_ITMP2);
1183                         emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
1184                         emit_mov_reg_reg(cd, REG_ITMP1, d);
1185                         emit_store_dst(jd, iptr, d);
1186                         break;
1187
1188                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1189
1190                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1191                         emit_ishift(cd, SHIFT_SHL, iptr->sx.s23.s2.var, iptr);
1192                         break;
1193
1194                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1195                                       /* sx.val.i = constant                             */
1196
1197                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1198                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1199                         M_INTMOVE(s1, d);
1200                         M_ISLL_IMM(iptr->sx.val.i, d);
1201                         emit_store_dst(jd, iptr, d);
1202                         break;
1203
1204                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1205
1206                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1207                         emit_ishift(cd, SHIFT_SAR, iptr->sx.s23.s2.var, iptr);
1208                         break;
1209
1210                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1211                                       /* sx.val.i = constant                             */
1212
1213                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1214                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1215                         M_INTMOVE(s1, d);
1216                         M_ISRA_IMM(iptr->sx.val.i, d);
1217                         emit_store_dst(jd, iptr, d);
1218                         break;
1219
1220                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1221
1222                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1223                         emit_ishift(cd, SHIFT_SHR, iptr->sx.s23.s2.var, iptr);
1224                         break;
1225
1226                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1227                                       /* sx.val.i = constant                             */
1228
1229                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1230                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1231                         M_INTMOVE(s1, d);
1232                         M_ISRL_IMM(iptr->sx.val.i, d);
1233                         emit_store_dst(jd, iptr, d);
1234                         break;
1235
1236                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1237
1238                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1239                         emit_lshift(cd, SHIFT_SHL, iptr->sx.s23.s2.var, iptr);
1240                         break;
1241
1242         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1243                                           /* sx.val.i = constant                             */
1244
1245                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1246                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1247                         M_INTMOVE(s1, d);
1248                         M_LSLL_IMM(iptr->sx.val.i, d);
1249                         emit_store_dst(jd, iptr, d);
1250                         break;
1251
1252                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1253
1254                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1255                         emit_lshift(cd, SHIFT_SAR, iptr->sx.s23.s2.var, iptr);
1256                         break;
1257
1258                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1259                                       /* sx.val.i = constant                             */
1260
1261                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1262                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1263                         M_INTMOVE(s1, d);
1264                         M_LSRA_IMM(iptr->sx.val.i, d);
1265                         emit_store_dst(jd, iptr, d);
1266                         break;
1267
1268                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1269
1270                         d = codegen_reg_of_dst(jd, iptr, REG_NULL);
1271                         emit_lshift(cd, SHIFT_SHR, iptr->sx.s23.s2.var, iptr);
1272                         break;
1273
1274                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1275                                       /* sx.val.l = constant                             */
1276
1277                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1278                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1279                         M_INTMOVE(s1, d);
1280                         M_LSRL_IMM(iptr->sx.val.i, d);
1281                         emit_store_dst(jd, iptr, d);
1282                         break;
1283
1284                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1285
1286                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1287                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1288                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1289                         if (s2 == d)
1290                                 M_IAND(s1, d);
1291                         else {
1292                                 M_INTMOVE(s1, d);
1293                                 M_IAND(s2, d);
1294                         }
1295                         emit_store_dst(jd, iptr, d);
1296                         break;
1297
1298                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1299                                       /* sx.val.i = constant                             */
1300
1301                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1302                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1303                         M_INTMOVE(s1, d);
1304                         M_IAND_IMM(iptr->sx.val.i, d);
1305                         emit_store_dst(jd, iptr, d);
1306                         break;
1307
1308                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1309
1310                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1311                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1312                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1313                         if (s2 == d)
1314                                 M_LAND(s1, d);
1315                         else {
1316                                 M_INTMOVE(s1, d);
1317                                 M_LAND(s2, d);
1318                         }
1319                         emit_store_dst(jd, iptr, d);
1320                         break;
1321
1322                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1323                                       /* sx.val.l = constant                             */
1324
1325                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1326                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1327                         M_INTMOVE(s1, d);
1328                         if (IS_IMM32(iptr->sx.val.l))
1329                                 M_LAND_IMM(iptr->sx.val.l, d);
1330                         else {
1331                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1332                                 M_LAND(REG_ITMP2, d);
1333                         }
1334                         emit_store_dst(jd, iptr, d);
1335                         break;
1336
1337                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1338
1339                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1340                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1341                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1342                         if (s2 == d)
1343                                 M_IOR(s1, d);
1344                         else {
1345                                 M_INTMOVE(s1, d);
1346                                 M_IOR(s2, d);
1347                         }
1348                         emit_store_dst(jd, iptr, d);
1349                         break;
1350
1351                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1352                                       /* sx.val.i = constant                             */
1353
1354                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1355                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1356                         M_INTMOVE(s1, d);
1357                         M_IOR_IMM(iptr->sx.val.i, d);
1358                         emit_store_dst(jd, iptr, d);
1359                         break;
1360
1361                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1362
1363                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1364                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1365                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1366                         if (s2 == d)
1367                                 M_LOR(s1, d);
1368                         else {
1369                                 M_INTMOVE(s1, d);
1370                                 M_LOR(s2, d);
1371                         }
1372                         emit_store_dst(jd, iptr, d);
1373                         break;
1374
1375                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1376                                       /* sx.val.l = constant                             */
1377
1378                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1379                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1380                         M_INTMOVE(s1, d);
1381                         if (IS_IMM32(iptr->sx.val.l))
1382                                 M_LOR_IMM(iptr->sx.val.l, d);
1383                         else {
1384                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1385                                 M_LOR(REG_ITMP2, d);
1386                         }
1387                         emit_store_dst(jd, iptr, d);
1388                         break;
1389
1390                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1391
1392                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1393                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1394                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1395                         if (s2 == d)
1396                                 M_IXOR(s1, d);
1397                         else {
1398                                 M_INTMOVE(s1, d);
1399                                 M_IXOR(s2, d);
1400                         }
1401                         emit_store_dst(jd, iptr, d);
1402                         break;
1403
1404                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1405                                       /* sx.val.i = constant                             */
1406
1407                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1408                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1409                         M_INTMOVE(s1, d);
1410                         M_IXOR_IMM(iptr->sx.val.i, d);
1411                         emit_store_dst(jd, iptr, d);
1412                         break;
1413
1414                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1415
1416                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1417                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1418                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
1419                         if (s2 == d)
1420                                 M_LXOR(s1, d);
1421                         else {
1422                                 M_INTMOVE(s1, d);
1423                                 M_LXOR(s2, d);
1424                         }
1425                         emit_store_dst(jd, iptr, d);
1426                         break;
1427
1428                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1429                                       /* sx.val.l = constant                             */
1430
1431                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1432                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1433                         M_INTMOVE(s1, d);
1434                         if (IS_IMM32(iptr->sx.val.l))
1435                                 M_LXOR_IMM(iptr->sx.val.l, d);
1436                         else {
1437                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
1438                                 M_LXOR(REG_ITMP2, d);
1439                         }
1440                         emit_store_dst(jd, iptr, d);
1441                         break;
1442
1443
1444                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1445                                       /* s1.localindex = variable, sx.val.i = constant             */
1446
1447                         var = &(rd->locals[iptr->s1.localindex][TYPE_INT]);
1448                         if (var->flags & INMEMORY) {
1449                                 s1 = REG_ITMP1;
1450                                 M_ILD(s1, REG_SP, var->regoff * 8);
1451                         } else
1452                                 s1 = var->regoff;
1453
1454                         /* Using inc and dec is not faster than add (tested with
1455                            sieve). */
1456
1457                         M_IADD_IMM(iptr->sx.val.i, s1);
1458
1459                         if (var->flags & INMEMORY)
1460                                 M_IST(s1, REG_SP, var->regoff * 8);
1461                         break;
1462
1463
1464                 /* floating operations ************************************************/
1465
1466                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1467
1468                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1469                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1470                         disp = dseg_adds4(cd, 0x80000000);
1471                         M_FLTMOVE(s1, d);
1472                         emit_movss_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1473                         emit_xorps_reg_reg(cd, REG_FTMP2, d);
1474                         emit_store_dst(jd, iptr, d);
1475                         break;
1476
1477                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1478
1479                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1480                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1481                         disp = dseg_adds8(cd, 0x8000000000000000);
1482                         M_FLTMOVE(s1, d);
1483                         emit_movd_membase_reg(cd, RIP, -((cd->mcodeptr + 9) - cd->mcodebase) + disp, REG_FTMP2);
1484                         emit_xorpd_reg_reg(cd, REG_FTMP2, d);
1485                         emit_store_dst(jd, iptr, d);
1486                         break;
1487
1488                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1489
1490                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1491                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1492                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1493                         if (s2 == d)
1494                                 M_FADD(s1, d);
1495                         else {
1496                                 M_FLTMOVE(s1, d);
1497                                 M_FADD(s2, d);
1498                         }
1499                         emit_store_dst(jd, iptr, d);
1500                         break;
1501
1502                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1503
1504                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1505                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1506                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1507                         if (s2 == d)
1508                                 M_DADD(s1, d);
1509                         else {
1510                                 M_FLTMOVE(s1, d);
1511                                 M_DADD(s2, d);
1512                         }
1513                         emit_store_dst(jd, iptr, d);
1514                         break;
1515
1516                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1517
1518                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1519                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1520                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1521                         if (s2 == d) {
1522                                 M_FLTMOVE(s2, REG_FTMP2);
1523                                 s2 = REG_FTMP2;
1524                         }
1525                         M_FLTMOVE(s1, d);
1526                         M_FSUB(s2, d);
1527                         emit_store_dst(jd, iptr, d);
1528                         break;
1529
1530                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1531
1532                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1533                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1534                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1535                         if (s2 == d) {
1536                                 M_FLTMOVE(s2, REG_FTMP2);
1537                                 s2 = REG_FTMP2;
1538                         }
1539                         M_FLTMOVE(s1, d);
1540                         M_DSUB(s2, d);
1541                         emit_store_dst(jd, iptr, d);
1542                         break;
1543
1544                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1545
1546                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1547                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1548                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1549                         if (s2 == d)
1550                                 M_FMUL(s1, d);
1551                         else {
1552                                 M_FLTMOVE(s1, d);
1553                                 M_FMUL(s2, d);
1554                         }
1555                         emit_store_dst(jd, iptr, d);
1556                         break;
1557
1558                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1559
1560                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1561                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1562                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1563                         if (s2 == d)
1564                                 M_DMUL(s1, d);
1565                         else {
1566                                 M_FLTMOVE(s1, d);
1567                                 M_DMUL(s2, d);
1568                         }
1569                         emit_store_dst(jd, iptr, d);
1570                         break;
1571
1572                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1573
1574                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1575                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1576                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1577                         if (s2 == d) {
1578                                 M_FLTMOVE(s2, REG_FTMP2);
1579                                 s2 = REG_FTMP2;
1580                         }
1581                         M_FLTMOVE(s1, d);
1582                         M_FDIV(s2, d);
1583                         emit_store_dst(jd, iptr, d);
1584                         break;
1585
1586                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1587
1588                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1589                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1590                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1591                         if (s2 == d) {
1592                                 M_FLTMOVE(s2, REG_FTMP2);
1593                                 s2 = REG_FTMP2;
1594                         }
1595                         M_FLTMOVE(s1, d);
1596                         M_DDIV(s2, d);
1597                         emit_store_dst(jd, iptr, d);
1598                         break;
1599
1600                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1601
1602                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1603                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1604                         M_CVTIF(s1, d);
1605                         emit_store_dst(jd, iptr, d);
1606                         break;
1607
1608                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1609
1610                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1611                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1612                         M_CVTID(s1, d);
1613                         emit_store_dst(jd, iptr, d);
1614                         break;
1615
1616                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1617
1618                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1619                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1620                         M_CVTLF(s1, d);
1621                         emit_store_dst(jd, iptr, d);
1622                         break;
1623
1624                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1625
1626                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1627                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
1628                         M_CVTLD(s1, d);
1629                         emit_store_dst(jd, iptr, d);
1630                         break;
1631
1632                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1633
1634                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1635                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1636                         M_CVTFI(s1, d);
1637                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1638                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1639                                 ((REG_RESULT == d) ? 0 : 3);
1640                         M_BNE(disp);
1641                         M_FLTMOVE(s1, REG_FTMP1);
1642                         M_MOV_IMM(asm_builtin_f2i, REG_ITMP2);
1643                         M_CALL(REG_ITMP2);
1644                         M_INTMOVE(REG_RESULT, d);
1645                         emit_store_dst(jd, iptr, d);
1646                         break;
1647
1648                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1649
1650                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1651                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1652                         M_CVTDI(s1, d);
1653                         M_ICMP_IMM(0x80000000, d);                        /* corner cases */
1654                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1655                                 ((REG_RESULT == d) ? 0 : 3);
1656                         M_BNE(disp);
1657                         M_FLTMOVE(s1, REG_FTMP1);
1658                         M_MOV_IMM(asm_builtin_d2i, REG_ITMP2);
1659                         M_CALL(REG_ITMP2);
1660                         M_INTMOVE(REG_RESULT, d);
1661                         emit_store_dst(jd, iptr, d);
1662                         break;
1663
1664                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1665
1666                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1667                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1668                         M_CVTFL(s1, d);
1669                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1670                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1671                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1672                                 ((REG_RESULT == d) ? 0 : 3);
1673                         M_BNE(disp);
1674                         M_FLTMOVE(s1, REG_FTMP1);
1675                         M_MOV_IMM(asm_builtin_f2l, REG_ITMP2);
1676                         M_CALL(REG_ITMP2);
1677                         M_INTMOVE(REG_RESULT, d);
1678                         emit_store_dst(jd, iptr, d);
1679                         break;
1680
1681                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1682
1683                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1684                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
1685                         M_CVTDL(s1, d);
1686                         M_MOV_IMM(0x8000000000000000, REG_ITMP2);
1687                         M_LCMP(REG_ITMP2, d);                             /* corner cases */
1688                         disp = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 +
1689                                 ((REG_RESULT == d) ? 0 : 3);
1690                         M_BNE(disp);
1691                         M_FLTMOVE(s1, REG_FTMP1);
1692                         M_MOV_IMM(asm_builtin_d2l, REG_ITMP2);
1693                         M_CALL(REG_ITMP2);
1694                         M_INTMOVE(REG_RESULT, d);
1695                         emit_store_dst(jd, iptr, d);
1696                         break;
1697
1698                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1699
1700                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1701                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1702                         M_CVTFD(s1, d);
1703                         emit_store_dst(jd, iptr, d);
1704                         break;
1705
1706                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1707
1708                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1709                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1710                         M_CVTDF(s1, d);
1711                         emit_store_dst(jd, iptr, d);
1712                         break;
1713
1714                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1715                                           /* == => 0, < => 1, > => -1 */
1716
1717                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1718                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1719                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1720                         M_CLR(d);
1721                         M_MOV_IMM(1, REG_ITMP1);
1722                         M_MOV_IMM(-1, REG_ITMP2);
1723                         emit_ucomiss_reg_reg(cd, s1, s2);
1724                         M_CMOVB(REG_ITMP1, d);
1725                         M_CMOVA(REG_ITMP2, d);
1726                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1727                         emit_store_dst(jd, iptr, d);
1728                         break;
1729
1730                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1731                                           /* == => 0, < => 1, > => -1 */
1732
1733                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1734                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1735                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1736                         M_CLR(d);
1737                         M_MOV_IMM(1, REG_ITMP1);
1738                         M_MOV_IMM(-1, REG_ITMP2);
1739                         emit_ucomiss_reg_reg(cd, s1, s2);
1740                         M_CMOVB(REG_ITMP1, d);
1741                         M_CMOVA(REG_ITMP2, d);
1742                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1743                         emit_store_dst(jd, iptr, d);
1744                         break;
1745
1746                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1747                                           /* == => 0, < => 1, > => -1 */
1748
1749                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1750                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1751                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1752                         M_CLR(d);
1753                         M_MOV_IMM(1, REG_ITMP1);
1754                         M_MOV_IMM(-1, REG_ITMP2);
1755                         emit_ucomisd_reg_reg(cd, s1, s2);
1756                         M_CMOVB(REG_ITMP1, d);
1757                         M_CMOVA(REG_ITMP2, d);
1758                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1759                         emit_store_dst(jd, iptr, d);
1760                         break;
1761
1762                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1763                                           /* == => 0, < => 1, > => -1 */
1764
1765                         s1 = emit_load_s1(jd, iptr, REG_FTMP1);
1766                         s2 = emit_load_s2(jd, iptr, REG_FTMP2);
1767                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1768                         M_CLR(d);
1769                         M_MOV_IMM(1, REG_ITMP1);
1770                         M_MOV_IMM(-1, REG_ITMP2);
1771                         emit_ucomisd_reg_reg(cd, s1, s2);
1772                         M_CMOVB(REG_ITMP1, d);
1773                         M_CMOVA(REG_ITMP2, d);
1774                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1775                         emit_store_dst(jd, iptr, d);
1776                         break;
1777
1778
1779                 /* memory operations **************************************************/
1780
1781                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1782
1783                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1784                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1785                         gen_nullptr_check(s1);
1786                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
1787                         emit_store_dst(jd, iptr, d);
1788                         break;
1789
1790                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
1791
1792                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1793                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1794                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1795                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1796                                 gen_nullptr_check(s1);
1797                                 gen_bound_check;
1798                         }
1799                         emit_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
1800                         emit_store_dst(jd, iptr, d);
1801                         break;
1802
1803                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
1804
1805                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1806                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1807                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1808                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1809                                 gen_nullptr_check(s1);
1810                                 gen_bound_check;
1811                         }
1812                         emit_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
1813                         emit_store_dst(jd, iptr, d);
1814                         break;
1815
1816                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
1817
1818                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1819                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1820                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1821                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1822                                 gen_nullptr_check(s1);
1823                                 gen_bound_check;
1824                         }
1825                         emit_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
1826                         emit_store_dst(jd, iptr, d);
1827                         break;
1828
1829                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
1830
1831                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1832                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1833                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1834                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1835                                 gen_nullptr_check(s1);
1836                                 gen_bound_check;
1837                         }
1838                         emit_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
1839                         emit_store_dst(jd, iptr, d);
1840                         break;
1841
1842                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1843
1844                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1845                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1846                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1847                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1848                                 gen_nullptr_check(s1);
1849                                 gen_bound_check;
1850                         }
1851                         emit_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
1852                         emit_store_dst(jd, iptr, d);
1853                         break;
1854
1855                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
1856
1857                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1858                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1859                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1860                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1861                                 gen_nullptr_check(s1);
1862                                 gen_bound_check;
1863                         }
1864                         emit_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
1865                         emit_store_dst(jd, iptr, d);
1866                         break;
1867
1868                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
1869
1870                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1871                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1872                         d = codegen_reg_of_dst(jd, iptr, REG_FTMP3);
1873                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1874                                 gen_nullptr_check(s1);
1875                                 gen_bound_check;
1876                         }
1877                         emit_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
1878                         emit_store_dst(jd, iptr, d);
1879                         break;
1880
1881                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1882
1883                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1884                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1885                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
1886                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1887                                 gen_nullptr_check(s1);
1888                                 gen_bound_check;
1889                         }
1890                         emit_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1891                         emit_store_dst(jd, iptr, d);
1892                         break;
1893
1894
1895                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
1896
1897                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1898                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1899                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1900                                 gen_nullptr_check(s1);
1901                                 gen_bound_check;
1902                         }
1903                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1904                         emit_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
1905                         break;
1906
1907                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
1908
1909                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1910                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1911                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1912                                 gen_nullptr_check(s1);
1913                                 gen_bound_check;
1914                         }
1915                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1916                         emit_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
1917                         break;
1918
1919                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
1920
1921                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1922                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1923                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1924                                 gen_nullptr_check(s1);
1925                                 gen_bound_check;
1926                         }
1927                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1928                         emit_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
1929                         break;
1930
1931                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
1932
1933                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1934                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1935                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1936                                 gen_nullptr_check(s1);
1937                                 gen_bound_check;
1938                         }
1939                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1940                         emit_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
1941                         break;
1942
1943                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
1944
1945                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1946                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1947                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1948                                 gen_nullptr_check(s1);
1949                                 gen_bound_check;
1950                         }
1951                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1952                         emit_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
1953                         break;
1954
1955                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
1956
1957                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1958                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1959                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1960                                 gen_nullptr_check(s1);
1961                                 gen_bound_check;
1962                         }
1963                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1964                         emit_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
1965                         break;
1966
1967                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
1968
1969                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1970                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1971                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1972                                 gen_nullptr_check(s1);
1973                                 gen_bound_check;
1974                         }
1975                         s3 = emit_load_s3(jd, iptr, REG_FTMP3);
1976                         emit_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
1977                         break;
1978
1979                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
1980
1981                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1982                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1983                         if (INSTRUCTION_MUST_CHECK(iptr)) {
1984                                 gen_nullptr_check(s1);
1985                                 gen_bound_check;
1986                         }
1987                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
1988
1989                         M_MOV(s1, rd->argintregs[0]);
1990                         M_MOV(s3, rd->argintregs[1]);
1991                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
1992                         M_CALL(REG_ITMP1);
1993                         M_TEST(REG_RESULT);
1994                         M_BEQ(0);
1995                         codegen_add_arraystoreexception_ref(cd);
1996
1997                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
1998                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
1999                         s3 = emit_load_s3(jd, iptr, REG_ITMP3);
2000                         emit_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2001                         break;
2002
2003
2004                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2005
2006                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2007                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2008                         if (INSTRUCTION_MUST_CHECK(iptr)) {
2009                                 gen_nullptr_check(s1);
2010                                 gen_bound_check;
2011                         }
2012                         emit_movb_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2013                         break;
2014
2015                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2016
2017                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2018                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2019                         if (INSTRUCTION_MUST_CHECK(iptr)) {
2020                                 gen_nullptr_check(s1);
2021                                 gen_bound_check;
2022                         }
2023                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_chararray, data[0]), s1, s2, 1);
2024                         break;
2025
2026                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2027
2028                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2029                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2030                         if (INSTRUCTION_MUST_CHECK(iptr)) {
2031                                 gen_nullptr_check(s1);
2032                                 gen_bound_check;
2033                         }
2034                         emit_movw_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2035                         break;
2036
2037                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2038
2039                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2040                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2041                         if (INSTRUCTION_MUST_CHECK(iptr)) {
2042                                 gen_nullptr_check(s1);
2043                                 gen_bound_check;
2044                         }
2045                         emit_movl_imm_memindex(cd, iptr->sx.s23.s3.constval, OFFSET(java_intarray, data[0]), s1, s2, 2);
2046                         break;
2047
2048                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2049
2050                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2051                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2052                         if (INSTRUCTION_MUST_CHECK(iptr)) {
2053                                 gen_nullptr_check(s1);
2054                                 gen_bound_check;
2055                         }
2056
2057                         if (IS_IMM32(iptr->sx.s23.s3.constval)) {
2058                                 emit_mov_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2059                         } else {
2060                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2061                                 emit_movl_imm_memindex(cd, (u4) (iptr->sx.s23.s3.constval >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2062                         }
2063                         break;
2064
2065                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2066
2067                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2068                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2069                         if (INSTRUCTION_MUST_CHECK(iptr)) {
2070                                 gen_nullptr_check(s1);
2071                                 gen_bound_check;
2072                         }
2073                         emit_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2074                         break;
2075
2076
2077                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2078
2079                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2080                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2081
2082                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2083                                 disp = dseg_addaddress(cd, NULL);
2084                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2085
2086                                 /* must be calculated before codegen_addpatchref */
2087
2088                                 if (opt_showdisassemble)
2089                                         disp -= PATCHER_CALL_SIZE;
2090
2091 /*                              PROFILE_CYCLE_STOP; */
2092
2093                                 codegen_addpatchref(cd, PATCHER_get_putstatic, uf, disp);
2094
2095                                 if (opt_showdisassemble) {
2096                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2097                                 }
2098
2099 /*                              PROFILE_CYCLE_START; */
2100                         }
2101                         else {
2102                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2103
2104                                 fieldtype = fi->type;
2105
2106                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2107                                         PROFILE_CYCLE_STOP;
2108
2109                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2110
2111                                         if (opt_showdisassemble) {
2112                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2113                                         }
2114
2115                                         PROFILE_CYCLE_START;
2116                                 }
2117
2118                                 disp = dseg_addaddress(cd, &(fi->value));
2119                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2120                         }
2121
2122                         /* This approach is much faster than moving the field
2123                            address inline into a register. */
2124
2125                         M_ALD(REG_ITMP1, RIP, disp);
2126
2127                         switch (fieldtype) {
2128                         case TYPE_INT:
2129                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2130                                 M_ILD(d, REG_ITMP1, 0);
2131                                 break;
2132                         case TYPE_LNG:
2133                         case TYPE_ADR:
2134                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
2135                                 M_LLD(d, REG_ITMP1, 0);
2136                                 break;
2137                         case TYPE_FLT:
2138                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2139                                 M_FLD(d, REG_ITMP1, 0);
2140                                 break;
2141                         case TYPE_DBL:
2142                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2143                                 M_DLD(d, REG_ITMP1, 0);
2144                                 break;
2145                         }
2146                         emit_store_dst(jd, iptr, d);
2147                         break;
2148
2149                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2150
2151                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2152                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2153
2154                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2155                                 disp = dseg_addaddress(cd, NULL);
2156                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2157
2158                                 /* must be calculated before codegen_addpatchref */
2159
2160                                 if (opt_showdisassemble)
2161                                         disp -= PATCHER_CALL_SIZE;
2162
2163 /*                              PROFILE_CYCLE_STOP; */
2164
2165                                 codegen_addpatchref(cd, PATCHER_get_putstatic, uf, disp);
2166
2167                                 if (opt_showdisassemble) {
2168                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2169                                 }
2170
2171 /*                              PROFILE_CYCLE_START; */
2172                         }
2173                         else {
2174                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2175
2176                                 fieldtype = fi->type;
2177
2178                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2179                                         PROFILE_CYCLE_STOP;
2180
2181                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2182
2183                                         if (opt_showdisassemble) {
2184                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2185                                                 disp -= PATCHER_CALL_SIZE;
2186                                         }
2187
2188                                         PROFILE_CYCLE_START;
2189                                 }
2190
2191                                 disp = dseg_addaddress(cd, &(fi->value));
2192                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2193                         }
2194
2195                         /* This approach is much faster than moving the field
2196                            address inline into a register. */
2197
2198                         M_ALD(REG_ITMP1, RIP, disp);
2199
2200                         switch (fieldtype) {
2201                         case TYPE_INT:
2202                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2203                                 M_IST(s1, REG_ITMP1, 0);
2204                                 break;
2205                         case TYPE_LNG:
2206                         case TYPE_ADR:
2207                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
2208                                 M_LST(s1, REG_ITMP1, 0);
2209                                 break;
2210                         case TYPE_FLT:
2211                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
2212                                 M_FST(s1, REG_ITMP1, 0);
2213                                 break;
2214                         case TYPE_DBL:
2215                                 s1 = emit_load_s1(jd, iptr, REG_FTMP1);
2216                                 M_DST(s1, REG_ITMP1, 0);
2217                                 break;
2218                         }
2219                         break;
2220
2221                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2222                                           /* val = value (in current instruction)     */
2223                                           /* following NOP)                           */
2224
2225                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2226                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2227
2228                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2229                                 disp = dseg_addaddress(cd, NULL);
2230                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2231
2232                                 /* must be calculated before codegen_addpatchref */
2233
2234                                 if (opt_showdisassemble)
2235                                         disp -= PATCHER_CALL_SIZE;
2236
2237
2238 /*                              PROFILE_CYCLE_STOP; */
2239
2240                                 codegen_addpatchref(cd, PATCHER_get_putstatic, uf, disp);
2241
2242                                 if (opt_showdisassemble) {
2243                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2244                                 }
2245
2246 /*                              PROFILE_CYCLE_START; */
2247                         }
2248                         else {
2249                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2250
2251                                 fieldtype = fi->type;
2252
2253                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2254                                         PROFILE_CYCLE_STOP;
2255
2256                                         codegen_addpatchref(cd, PATCHER_clinit, fi->class, 0);
2257
2258                                         if (opt_showdisassemble) {
2259                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2260                                         }
2261
2262                                         PROFILE_CYCLE_START;
2263                                 }
2264
2265                                 disp = dseg_addaddress(cd, &(fi->value));
2266                                 disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
2267                         }
2268
2269                         /* This approach is much faster than moving the field
2270                            address inline into a register. */
2271
2272                         M_ALD(REG_ITMP1, RIP, disp);
2273
2274                         switch (fieldtype) {
2275                         case TYPE_INT:
2276                         case TYPE_FLT:
2277                                 M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2278                                 break;
2279                         case TYPE_LNG:
2280                         case TYPE_ADR:
2281                         case TYPE_DBL:
2282                                 if (IS_IMM32(iptr->sx.s23.s2.constval))
2283                                         M_LST_IMM32(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2284                                 else {
2285                                         M_IST_IMM(iptr->sx.s23.s2.constval, REG_ITMP1, 0);
2286                                         M_IST_IMM(iptr->sx.s23.s2.constval >> 32, REG_ITMP1, 4);
2287                                 }
2288                                 break;
2289                         }
2290                         break;
2291
2292                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2293
2294                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2295                         gen_nullptr_check(s1);
2296
2297                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2298                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2299
2300 /*                              PROFILE_CYCLE_STOP; */
2301
2302                                 codegen_addpatchref(cd, PATCHER_get_putfield, uf, 0);
2303
2304                                 if (opt_showdisassemble) {
2305                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2306                                 }
2307
2308 /*                              PROFILE_CYCLE_START; */
2309
2310                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2311                                 disp = 0;
2312                         }
2313                         else {
2314                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2315
2316                                 fieldtype = fi->type;
2317                                 disp = fi->offset;
2318                         }
2319
2320                         switch (fieldtype) {
2321                         case TYPE_INT:
2322                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
2323                                 M_ILD32(d, s1, disp);
2324                                 break;
2325                         case TYPE_LNG:
2326                         case TYPE_ADR:
2327                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP1);
2328                                 M_LLD32(d, s1, disp);
2329                                 break;
2330                         case TYPE_FLT:
2331                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2332                                 M_FLD32(d, s1, disp);
2333                                 break;
2334                         case TYPE_DBL:
2335                                 d = codegen_reg_of_dst(jd, iptr, REG_FTMP1);
2336                                 M_DLD32(d, s1, disp);
2337                                 break;
2338                         }
2339                         emit_store_dst(jd, iptr, d);
2340                         break;
2341
2342                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2343
2344                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2345                         gen_nullptr_check(s1);
2346
2347                         s2 = emit_load_s2(jd, iptr, REG_IFTMP);
2348
2349                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2350                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2351
2352 /*                              PROFILE_CYCLE_STOP; */
2353
2354                                 codegen_addpatchref(cd, PATCHER_get_putfield, uf, 0);
2355
2356                                 if (opt_showdisassemble) {
2357                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2358                                 }
2359
2360 /*                              PROFILE_CYCLE_START; */
2361
2362                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2363                                 disp = 0;
2364                         }
2365                         else {
2366                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2367
2368                                 fieldtype = fi->type;
2369                                 disp = fi->offset;
2370                         }
2371
2372                         switch (fieldtype) {
2373                         case TYPE_INT:
2374                                 M_IST32(s2, s1, disp);
2375                                 break;
2376                         case TYPE_LNG:
2377                         case TYPE_ADR:
2378                                 M_LST32(s2, s1, disp);
2379                                 break;
2380                         case TYPE_FLT:
2381                                 M_FST32(s2, s1, disp);
2382                                 break;
2383                         case TYPE_DBL:
2384                                 M_DST32(s2, s1, disp);
2385                                 break;
2386                         }
2387                         break;
2388
2389                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2390                                           /* val = value (in current instruction)     */
2391                                           /* following NOP)                           */
2392
2393                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2394                         gen_nullptr_check(s1);
2395
2396                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2397                                 unresolved_field *uf = iptr->sx.s23.s3.uf;
2398
2399 /*                              PROFILE_CYCLE_STOP; */
2400
2401                                 codegen_addpatchref(cd, PATCHER_putfieldconst, uf, 0);
2402
2403                                 if (opt_showdisassemble) {
2404                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2405                                 }
2406
2407 /*                              PROFILE_CYCLE_START; */
2408
2409                                 fieldtype = uf->fieldref->parseddesc.fd->type;
2410                                 disp = 0;
2411                         }
2412                         else {
2413                                 fieldinfo *fi = iptr->sx.s23.s3.fmiref->p.field;
2414
2415                                 fieldtype = fi->type;
2416                                 disp = fi->offset;
2417                         }
2418
2419                         switch (fieldtype) {
2420                         case TYPE_INT:
2421                         case TYPE_FLT:
2422                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2423                                 break;
2424                         case TYPE_LNG:
2425                         case TYPE_ADR:
2426                         case TYPE_DBL:
2427                                 /* XXX why no check for IS_IMM32? */
2428                                 M_IST32_IMM(iptr->sx.s23.s2.constval, s1, disp);
2429                                 M_IST32_IMM(iptr->sx.s23.s2.constval >> 32, s1, disp + 4);
2430                                 break;
2431                         }
2432                         break;
2433
2434
2435                 /* branch operations **************************************************/
2436
2437                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2438
2439                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2440                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2441
2442                         PROFILE_CYCLE_STOP;
2443
2444 #ifdef ENABLE_VERIFIER
2445                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2446                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2447                                                                         iptr->sx.s23.s2.uc, 0);
2448
2449                                 if (opt_showdisassemble) {
2450                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2451                                 }
2452                         }
2453 #endif /* ENABLE_VERIFIER */
2454
2455                         M_CALL_IMM(0);                            /* passing exception pc */
2456                         M_POP(REG_ITMP2_XPC);
2457
2458                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2459                         M_JMP(REG_ITMP3);
2460                         break;
2461
2462                 case ICMD_GOTO:         /* ... ==> ...                                */
2463
2464                         M_JMP_IMM(0);
2465                         codegen_addreference(cd, iptr->dst.block);
2466                         break;
2467
2468                 case ICMD_JSR:          /* ... ==> ...                                */
2469
2470                         M_CALL_IMM(0);
2471                         codegen_addreference(cd, iptr->sx.s23.s3.jsrtarget.block);
2472                         break;
2473
2474                 case ICMD_RET:          /* ... ==> ...                                */
2475                                         /* s1.localindex = local variable                       */
2476
2477                         var = &(rd->locals[iptr->s1.localindex][TYPE_ADR]);
2478                         if (var->flags & INMEMORY) {
2479                                 M_ALD(REG_ITMP1, REG_SP, var->regoff * 8);
2480                                 M_JMP(REG_ITMP1);
2481                         } else
2482                                 M_JMP(var->regoff);
2483                         break;
2484
2485                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2486
2487                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2488                         M_TEST(s1);
2489                         M_BEQ(0);
2490                         codegen_addreference(cd, iptr->dst.block);
2491                         break;
2492
2493                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2494
2495                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2496                         M_TEST(s1);
2497                         M_BNE(0);
2498                         codegen_addreference(cd, iptr->dst.block);
2499                         break;
2500
2501                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2502
2503                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2504                         M_ICMP_IMM(iptr->sx.val.i, s1);
2505                         M_BEQ(0);
2506                         codegen_addreference(cd, iptr->dst.block);
2507                         break;
2508
2509                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2510
2511                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2512                         M_ICMP_IMM(iptr->sx.val.i, s1);
2513                         M_BLT(0);
2514                         codegen_addreference(cd, iptr->dst.block);
2515                         break;
2516
2517                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2518
2519                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2520                         M_ICMP_IMM(iptr->sx.val.i, s1);
2521                         M_BLE(0);
2522                         codegen_addreference(cd, iptr->dst.block);
2523                         break;
2524
2525                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2526
2527                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2528                         M_ICMP_IMM(iptr->sx.val.i, s1);
2529                         M_BNE(0);
2530                         codegen_addreference(cd, iptr->dst.block);
2531                         break;
2532
2533                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2534
2535                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2536                         M_ICMP_IMM(iptr->sx.val.i, s1);
2537                         M_BGT(0);
2538                         codegen_addreference(cd, iptr->dst.block);
2539                         break;
2540
2541                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2542
2543                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2544                         M_ICMP_IMM(iptr->sx.val.i, s1);
2545                         M_BGE(0);
2546                         codegen_addreference(cd, iptr->dst.block);
2547                         break;
2548
2549                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2550
2551                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2552                         if (IS_IMM32(iptr->sx.val.l))
2553                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2554                         else {
2555                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2556                                 M_LCMP(REG_ITMP2, s1);
2557                         }
2558                         M_BEQ(0);
2559                         codegen_addreference(cd, iptr->dst.block);
2560                         break;
2561
2562                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2563
2564                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2565                         if (IS_IMM32(iptr->sx.val.l))
2566                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2567                         else {
2568                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2569                                 M_LCMP(REG_ITMP2, s1);
2570                         }
2571                         M_BLT(0);
2572                         codegen_addreference(cd, iptr->dst.block);
2573                         break;
2574
2575                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2576
2577                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2578                         if (IS_IMM32(iptr->sx.val.l))
2579                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2580                         else {
2581                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2582                                 M_LCMP(REG_ITMP2, s1);
2583                         }
2584                         M_BLE(0);
2585                         codegen_addreference(cd, iptr->dst.block);
2586                         break;
2587
2588                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2589
2590                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2591                         if (IS_IMM32(iptr->sx.val.l))
2592                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2593                         else {
2594                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2595                                 M_LCMP(REG_ITMP2, s1);
2596                         }
2597                         M_BNE(0);
2598                         codegen_addreference(cd, iptr->dst.block);
2599                         break;
2600
2601                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2602
2603                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2604                         if (IS_IMM32(iptr->sx.val.l))
2605                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2606                         else {
2607                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2608                                 M_LCMP(REG_ITMP2, s1);
2609                         }
2610                         M_BGT(0);
2611                         codegen_addreference(cd, iptr->dst.block);
2612                         break;
2613
2614                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2615
2616                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2617                         if (IS_IMM32(iptr->sx.val.l))
2618                                 M_LCMP_IMM(iptr->sx.val.l, s1);
2619                         else {
2620                                 M_MOV_IMM(iptr->sx.val.l, REG_ITMP2);
2621                                 M_LCMP(REG_ITMP2, s1);
2622                         }
2623                         M_BGE(0);
2624                         codegen_addreference(cd, iptr->dst.block);
2625                         break;
2626
2627                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2628
2629                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2630                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2631                         M_ICMP(s2, s1);
2632                         M_BEQ(0);
2633                         codegen_addreference(cd, iptr->dst.block);
2634                         break;
2635
2636                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2637                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2638
2639                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2640                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2641                         M_LCMP(s2, s1);
2642                         M_BEQ(0);
2643                         codegen_addreference(cd, iptr->dst.block);
2644                         break;
2645
2646                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2647
2648                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2649                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2650                         M_ICMP(s2, s1);
2651                         M_BNE(0);
2652                         codegen_addreference(cd, iptr->dst.block);
2653                         break;
2654
2655                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2656                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2657
2658                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2659                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2660                         M_LCMP(s2, s1);
2661                         M_BNE(0);
2662                         codegen_addreference(cd, iptr->dst.block);
2663                         break;
2664
2665                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2666
2667                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2668                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2669                         M_ICMP(s2, s1);
2670                         M_BLT(0);
2671                         codegen_addreference(cd, iptr->dst.block);
2672                         break;
2673
2674                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2675
2676                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2677                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2678                         M_LCMP(s2, s1);
2679                         M_BLT(0);
2680                         codegen_addreference(cd, iptr->dst.block);
2681                         break;
2682
2683                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2684
2685                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2686                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2687                         M_ICMP(s2, s1);
2688                         M_BGT(0);
2689                         codegen_addreference(cd, iptr->dst.block);
2690                         break;
2691
2692                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2693
2694                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2695                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2696                         M_LCMP(s2, s1);
2697                         M_BGT(0);
2698                         codegen_addreference(cd, iptr->dst.block);
2699                         break;
2700
2701                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2702
2703                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2704                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2705                         M_ICMP(s2, s1);
2706                         M_BLE(0);
2707                         codegen_addreference(cd, iptr->dst.block);
2708                         break;
2709
2710                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2711
2712                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2713                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2714                         M_LCMP(s2, s1);
2715                         M_BLE(0);
2716                         codegen_addreference(cd, iptr->dst.block);
2717                         break;
2718
2719                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2720
2721                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2722                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2723                         M_ICMP(s2, s1);
2724                         M_BGE(0);
2725                         codegen_addreference(cd, iptr->dst.block);
2726                         break;
2727
2728                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2729
2730                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2731                         s2 = emit_load_s2(jd, iptr, REG_ITMP2);
2732                         M_LCMP(s2, s1);
2733                         M_BGE(0);
2734                         codegen_addreference(cd, iptr->dst.block);
2735                         break;
2736
2737                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2738                 case ICMD_LRETURN:
2739
2740                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2741                         M_INTMOVE(s1, REG_RESULT);
2742                         goto nowperformreturn;
2743
2744                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2745
2746                         s1 = emit_load_s1(jd, iptr, REG_RESULT);
2747                         M_INTMOVE(s1, REG_RESULT);
2748
2749 #ifdef ENABLE_VERIFIER
2750                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2751                                 PROFILE_CYCLE_STOP;
2752
2753                                 codegen_addpatchref(cd, PATCHER_athrow_areturn,
2754                                                                         iptr->sx.s23.s2.uc, 0);
2755
2756                                 if (opt_showdisassemble) {
2757                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2758                                 }
2759
2760                                 PROFILE_CYCLE_START;
2761                         }
2762 #endif /* ENABLE_VERIFIER */
2763                         goto nowperformreturn;
2764
2765                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2766                 case ICMD_DRETURN:
2767
2768                         s1 = emit_load_s1(jd, iptr, REG_FRESULT);
2769                         M_FLTMOVE(s1, REG_FRESULT);
2770                         goto nowperformreturn;
2771
2772                 case ICMD_RETURN:      /* ...  ==> ...                                */
2773
2774 nowperformreturn:
2775                         {
2776                         s4 i, p;
2777
2778                         p = cd->stackframesize;
2779
2780 #if !defined(NDEBUG)
2781                         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
2782                                 emit_verbosecall_exit(jd);
2783 #endif /* !defined(NDEBUG) */
2784
2785 #if defined(ENABLE_THREADS)
2786                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2787                                 M_ALD(rd->argintregs[0], REG_SP, rd->memuse * 8);
2788
2789                                 /* we need to save the proper return value */
2790                                 switch (iptr->opc) {
2791                                 case ICMD_IRETURN:
2792                                 case ICMD_ARETURN:
2793                                 case ICMD_LRETURN:
2794                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2795                                         break;
2796                                 case ICMD_FRETURN:
2797                                 case ICMD_DRETURN:
2798                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2799                                         break;
2800                                 }
2801
2802                                 M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
2803                                 M_CALL(REG_ITMP1);
2804
2805                                 /* and now restore the proper return value */
2806                                 switch (iptr->opc) {
2807                                 case ICMD_IRETURN:
2808                                 case ICMD_ARETURN:
2809                                 case ICMD_LRETURN:
2810                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2811                                         break;
2812                                 case ICMD_FRETURN:
2813                                 case ICMD_DRETURN:
2814                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2815                                         break;
2816                                 }
2817                         }
2818 #endif
2819
2820                         /* restore saved registers */
2821
2822                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2823                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2824                         }
2825                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2826                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2827                         }
2828
2829                         /* deallocate stack */
2830
2831                         if (cd->stackframesize)
2832                                 M_AADD_IMM(cd->stackframesize * 8, REG_SP);
2833
2834                         /* generate method profiling code */
2835
2836                         PROFILE_CYCLE_STOP;
2837
2838                         M_RET;
2839                         }
2840                         break;
2841
2842
2843                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2844                         {
2845                                 s4 i, l;
2846                                 branch_target_t *table;
2847
2848                                 table = iptr->dst.table;
2849
2850                                 l = iptr->sx.s23.s2.tablelow;
2851                                 i = iptr->sx.s23.s3.tablehigh;
2852
2853                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2854                                 M_INTMOVE(s1, REG_ITMP1);
2855
2856                                 if (l != 0)
2857                                         M_ISUB_IMM(l, REG_ITMP1);
2858
2859                                 /* number of targets */
2860                                 i = i - l + 1;
2861
2862                 /* range check */
2863                                 M_ICMP_IMM(i - 1, REG_ITMP1);
2864                                 M_BA(0);
2865
2866                                 codegen_addreference(cd, table[0].block); /* default target */
2867
2868                                 /* build jump table top down and use address of lowest entry */
2869
2870                                 table += i;
2871
2872                                 while (--i >= 0) {
2873                                         dseg_addtarget(cd, table->block);
2874                                         --table;
2875                                 }
2876
2877                                 /* length of dataseg after last dseg_addtarget is used
2878                                    by load */
2879
2880                                 M_MOV_IMM(0, REG_ITMP2);
2881                                 dseg_adddata(cd);
2882                                 emit_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2883                                 M_JMP(REG_ITMP1);
2884                         }
2885                         break;
2886
2887
2888                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2889                         {
2890                                 s4 i;
2891                                 lookup_target_t *lookup;
2892
2893                                 lookup = iptr->dst.lookup;
2894
2895                                 i = iptr->sx.s23.s2.lookupcount;
2896
2897                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
2898                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
2899
2900                                 while (--i >= 0) {
2901                                         M_ICMP_IMM(lookup->value, s1);
2902                                         M_BEQ(0);
2903                                         codegen_addreference(cd, lookup->target.block);
2904                                         lookup++;
2905                                 }
2906
2907                                 M_JMP_IMM(0);
2908
2909                                 codegen_addreference(cd, iptr->sx.s23.s3.lookupdefault.block);
2910                         }
2911                         break;
2912
2913
2914                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
2915
2916                         bte = iptr->sx.s23.s3.bte;
2917                         md = bte->md;
2918
2919                         /* CHECKNULL for MONITORENTER/EXIT */
2920                         if (bte->opcode == ICMD_MONITORENTER || bte->opcode == ICMD_MONITOREXIT) {
2921                                 s1 = emit_load(jd, iptr, iptr->sx.s23.s2.args[0], REG_ITMP1);
2922                                 M_TEST(s1);
2923                                 M_BEQ(0);
2924                                 codegen_add_nullpointerexception_ref(cd);
2925                         }
2926
2927                         goto gen_method;
2928
2929                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2930
2931                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2932                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
2933                 case ICMD_INVOKEINTERFACE:
2934
2935                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
2936                                 md = iptr->sx.s23.s3.um->methodref->parseddesc.md;
2937                                 lm = NULL;
2938                         }
2939                         else {
2940                                 lm = iptr->sx.s23.s3.fmiref->p.method;
2941                                 md = lm->parseddesc;
2942                         }
2943
2944 gen_method:
2945                         s3 = md->paramcount;
2946
2947                         MCODECHECK((20 * s3) + 128);
2948
2949                         /* copy arguments to registers or stack location */
2950
2951                         for (s3 = s3 - 1; s3 >= 0; s3--) {
2952                                 src = iptr->sx.s23.s2.args[s3];
2953
2954                                 if (src->varkind == ARGVAR)
2955                                         continue;
2956                                 if (IS_INT_LNG_TYPE(src->type)) {
2957                                         if (!md->params[s3].inmemory) {
2958                                                 s1 = rd->argintregs[md->params[s3].regoff];
2959                                                 d = emit_load(jd, iptr, src, s1);
2960                                                 M_INTMOVE(d, s1);
2961                                         } else {
2962                                                 d = emit_load(jd, iptr, src, REG_ITMP1);
2963                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
2964                                         }
2965
2966                                 } else {
2967                                         if (!md->params[s3].inmemory) {
2968                                                 s1 = rd->argfltregs[md->params[s3].regoff];
2969                                                 d = emit_load(jd, iptr, src, s1);
2970                                                 M_FLTMOVE(d, s1);
2971                                         } else {
2972                                                 d = emit_load(jd, iptr, src, REG_FTMP1);
2973
2974                                                 if (IS_2_WORD_TYPE(src->type))
2975                                                         M_DST(d, REG_SP, md->params[s3].regoff * 8);
2976                                                 else
2977                                                         M_FST(d, REG_SP, md->params[s3].regoff * 8);
2978                                         }
2979                                 }
2980                         }
2981
2982                         /* generate method profiling code */
2983
2984                         PROFILE_CYCLE_STOP;
2985
2986                         switch (iptr->opc) {
2987                         case ICMD_BUILTIN:
2988                                 a = (ptrint) bte->fp;
2989                                 d = md->returntype.type;
2990
2991                                 M_MOV_IMM(a, REG_ITMP1);
2992                                 M_CALL(REG_ITMP1);
2993
2994
2995                                 if (INSTRUCTION_MUST_CHECK(iptr)) {
2996                                         M_TEST(REG_RESULT);
2997                                         M_BEQ(0);
2998                                         codegen_add_fillinstacktrace_ref(cd);
2999                                 }
3000                                 break;
3001
3002                         case ICMD_INVOKESPECIAL:
3003                                 M_TEST(rd->argintregs[0]);
3004                                 M_BEQ(0);
3005                                 codegen_add_nullpointerexception_ref(cd);
3006
3007                                 /* fall through */
3008
3009                         case ICMD_INVOKESTATIC:
3010                                 if (lm == NULL) {
3011                                         unresolved_method *um = iptr->sx.s23.s3.um;
3012
3013                                         disp = dseg_addaddress(cd, NULL);
3014                                         disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
3015
3016                                         /* must be calculated before codegen_addpatchref */
3017
3018                                         if (opt_showdisassemble)
3019                                                 disp -= PATCHER_CALL_SIZE;
3020
3021                                         codegen_addpatchref(cd, PATCHER_invokestatic_special,
3022                                                                                 um, disp);
3023
3024                                         if (opt_showdisassemble) {
3025                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3026                                         }
3027
3028                                         a = 0;
3029                                         d = um->methodref->parseddesc.md->returntype.type;
3030                                 }
3031                                 else {
3032                                         disp = dseg_addaddress(cd, lm->stubroutine);
3033                                         disp = -((cd->mcodeptr + 7) - cd->mcodebase) + disp;
3034
3035                                         a = (ptrint) lm->stubroutine;
3036                                         d = lm->parseddesc->returntype.type;
3037                                 }
3038
3039 /*                              M_MOV_IMM(a, REG_ITMP2); */
3040                                 M_ALD(REG_ITMP2, RIP, disp);
3041                                 M_CALL(REG_ITMP2);
3042                                 break;
3043
3044                         case ICMD_INVOKEVIRTUAL:
3045                                 gen_nullptr_check(rd->argintregs[0]);
3046
3047                                 if (lm == NULL) {
3048                                         unresolved_method *um = iptr->sx.s23.s3.um;
3049
3050                                         codegen_addpatchref(cd, PATCHER_invokevirtual, um, 0);
3051
3052                                         if (opt_showdisassemble) {
3053                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3054                                         }
3055
3056                                         s1 = 0;
3057                                         d = um->methodref->parseddesc.md->returntype.type;
3058
3059                                 } else {
3060                                         s1 = OFFSET(vftbl_t, table[0]) +
3061                                                 sizeof(methodptr) * lm->vftblindex;
3062                                         d = lm->parseddesc->returntype.type;
3063                                 }
3064
3065                                 M_ALD(REG_METHODPTR, rd->argintregs[0],
3066                                           OFFSET(java_objectheader, vftbl));
3067                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s1);
3068                                 M_CALL(REG_ITMP3);
3069                                 break;
3070
3071                         case ICMD_INVOKEINTERFACE:
3072                                 gen_nullptr_check(rd->argintregs[0]);
3073
3074                                 if (lm == NULL) {
3075                                         unresolved_method *um = iptr->sx.s23.s3.um;
3076
3077                                         codegen_addpatchref(cd, PATCHER_invokeinterface, um, 0);
3078
3079                                         if (opt_showdisassemble) {
3080                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3081                                         }
3082
3083                                         s1 = 0;
3084                                         s2 = 0;
3085                                         d = um->methodref->parseddesc.md->returntype.type;
3086
3087                                 } else {
3088                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3089                                                 sizeof(methodptr) * lm->class->index;
3090
3091                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3092
3093                                         d = lm->parseddesc->returntype.type;
3094                                 }
3095
3096                                 M_ALD(REG_METHODPTR, rd->argintregs[0],
3097                                           OFFSET(java_objectheader, vftbl));
3098                                 M_ALD32(REG_METHODPTR, REG_METHODPTR, s1);
3099                                 M_ALD32(REG_ITMP3, REG_METHODPTR, s2);
3100                                 M_CALL(REG_ITMP3);
3101                                 break;
3102                         }
3103
3104                         /* generate method profiling code */
3105
3106                         PROFILE_CYCLE_START;
3107
3108                         /* d contains return type */
3109
3110                         if (d != TYPE_VOID) {
3111                                 if (IS_INT_LNG_TYPE(iptr->dst.var->type)) {
3112                                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
3113                                         M_INTMOVE(REG_RESULT, s1);
3114                                         emit_store_dst(jd, iptr, s1);
3115                                 } else {
3116                                         s1 = codegen_reg_of_dst(jd, iptr, REG_FRESULT);
3117                                         M_FLTMOVE(REG_FRESULT, s1);
3118                                         emit_store_dst(jd, iptr, s1);
3119                                 }
3120                         }
3121                         break;
3122
3123
3124                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3125
3126                                       /* val.a: (classinfo *) superclass              */
3127
3128                         /*  superclass is an interface:
3129                          *
3130                          *  OK if ((sub == NULL) ||
3131                          *         (sub->vftbl->interfacetablelength > super->index) &&
3132                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3133                          *
3134                          *  superclass is a class:
3135                          *
3136                          *  OK if ((sub == NULL) || (0
3137                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3138                          *         super->vftbl->diffval));
3139                          */
3140
3141                         if (!(iptr->flags.bits & INS_FLAG_ARRAY)) {
3142                                 /* object type cast-check */
3143
3144                                 classinfo *super;
3145                                 vftbl_t   *supervftbl;
3146                                 s4         superindex;
3147
3148                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3149                                         super = NULL;
3150                                         superindex = 0;
3151                                         supervftbl = NULL;
3152                                 }
3153                                 else {
3154                                         super = iptr->sx.s23.s3.c.cls;
3155                                         superindex = super->index;
3156                                         supervftbl = super->vftbl;
3157                                 }
3158
3159 #if defined(ENABLE_THREADS)
3160                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3161 #endif
3162                                 s1 = emit_load_s1(jd, iptr, REG_ITMP1);
3163
3164                                 /* calculate interface checkcast code size */
3165
3166                                 s2 = 3; /* mov_membase_reg */
3167                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3168
3169                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3170                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3171                                         3 /* test */ + 6 /* jcc */;
3172
3173                                 if (super == NULL)
3174                                         s2 += (opt_showdisassemble ? 5 : 0);
3175
3176                                 /* calculate class checkcast code size */
3177
3178                                 s3 = 3; /* mov_membase_reg */
3179                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3180                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3181
3182 #if 0
3183                                 if (s1 != REG_ITMP1) {
3184                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3185                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3186                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3187                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3188                                         a += 3;    /* sub */
3189
3190                                 } else
3191 #endif
3192                                         {
3193                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3194                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3195                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3196                                         }
3197
3198                                 s3 += 3 /* cmp */ + 6 /* jcc */;
3199
3200                                 if (super == NULL)
3201                                         s3 += (opt_showdisassemble ? 5 : 0);
3202
3203                                 /* if class is not resolved, check which code to call */
3204
3205                                 if (super == NULL) {
3206                                         M_TEST(s1);
3207                                         M_BEQ(6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3208
3209                                         codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3210                                                                                 iptr->sx.s23.s3.c.ref, 0);
3211
3212                                         if (opt_showdisassemble) {
3213                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3214                                         }
3215
3216                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
3217                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
3218                                         M_BEQ(s2 + 5);
3219                                 }
3220
3221                                 /* interface checkcast code */
3222
3223                                 if ((super == NULL) || (super->flags & ACC_INTERFACE)) {
3224                                         if (super != NULL) {
3225                                                 M_TEST(s1);
3226                                                 M_BEQ(s2);
3227                                         }
3228
3229                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3230
3231                                         if (super == NULL) {
3232                                                 codegen_addpatchref(cd,
3233                                                                                         PATCHER_checkcast_instanceof_interface,
3234                                                                                         iptr->sx.s23.s3.c.ref,
3235                                                                                         0);
3236
3237                                                 if (opt_showdisassemble) {
3238                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3239                                                 }
3240                                         }
3241
3242                                         emit_movl_membase32_reg(cd, REG_ITMP2,
3243                                                                                           OFFSET(vftbl_t, interfacetablelength),
3244                                                                                           REG_ITMP3);
3245                                         /* XXX TWISTI: should this be int arithmetic? */
3246                                         M_LSUB_IMM32(superindex, REG_ITMP3);
3247                                         M_TEST(REG_ITMP3);
3248                                         M_BLE(0);
3249                                         codegen_add_classcastexception_ref(cd, s1);
3250                                         emit_mov_membase32_reg(cd, REG_ITMP2,
3251                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
3252                                                                                          superindex * sizeof(methodptr*),
3253                                                                                          REG_ITMP3);
3254                                         M_TEST(REG_ITMP3);
3255                                         M_BEQ(0);
3256                                         codegen_add_classcastexception_ref(cd, s1);
3257
3258                                         if (super == NULL)
3259                                                 M_JMP_IMM(s3);
3260                                 }
3261
3262                                 /* class checkcast code */
3263
3264                                 if ((super == NULL) || !(super->flags & ACC_INTERFACE)) {
3265                                         if (super != NULL) {
3266                                                 M_TEST(s1);
3267                                                 M_BEQ(s3);
3268                                         }
3269
3270                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3271
3272                                         if (super == NULL) {
3273                                                 codegen_addpatchref(cd, PATCHER_checkcast_class,
3274                                                                                         iptr->sx.s23.s3.c.ref,
3275                                                                                         0);
3276
3277                                                 if (opt_showdisassemble) {
3278                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3279                                                 }
3280                                         }
3281
3282                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3283 #if defined(ENABLE_THREADS)
3284                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3285 #endif
3286                                         emit_movl_membase32_reg(cd, REG_ITMP2,
3287                                                                                           OFFSET(vftbl_t, baseval),
3288                                                                                           REG_ITMP2);
3289                                         /*                                      if (s1 != REG_ITMP1) { */
3290                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3291                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
3292                                         /*                                                                                              REG_ITMP1); */
3293                                         /*                                              emit_movl_membase_reg(cd, REG_ITMP3, */
3294                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3295                                         /*                                                                                              REG_ITMP3); */
3296                                         /*  #if defined(ENABLE_THREADS) */
3297                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3298                                         /*  #endif */
3299                                         /*                                              emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP1, REG_ITMP2); */
3300
3301                                         /*                                      } else { */
3302                                         emit_movl_membase32_reg(cd, REG_ITMP3,
3303                                                                                           OFFSET(vftbl_t, baseval),
3304                                                                                           REG_ITMP3);
3305                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3306                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3307                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3308                                         /*                                      } */
3309 #if defined(ENABLE_THREADS)
3310                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3311 #endif
3312                                         M_LCMP(REG_ITMP3, REG_ITMP2);
3313                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3314                                         codegen_add_classcastexception_ref(cd, s1);
3315                                 }
3316
3317                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP3);
3318                         }
3319                         else {
3320                                 /* array type cast-check */
3321
3322                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
3323                                 M_INTMOVE(s1, rd->argintregs[0]);
3324
3325                                 if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3326                                         codegen_addpatchref(cd, PATCHER_builtin_arraycheckcast,
3327                                                                                 iptr->sx.s23.s3.c.ref, 0);
3328
3329                                         if (opt_showdisassemble) {
3330                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3331                                         }
3332                                 }
3333
3334                                 M_MOV_IMM(iptr->sx.s23.s3.c.cls, rd->argintregs[1]);
3335                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
3336                                 M_CALL(REG_ITMP1);
3337
3338                                 /* s1 may have been destroyed over the function call */
3339                                 s1 = emit_load_s1(jd, iptr, REG_ITMP2);
3340                                 M_TEST(REG_RESULT);
3341                                 M_BEQ(0);
3342                                 codegen_add_classcastexception_ref(cd, s1);
3343
3344                                 d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
3345                         }
3346
3347                         M_INTMOVE(s1, d);
3348                         emit_store_dst(jd, iptr, d);
3349                         break;
3350
3351                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3352
3353                                       /* val.a: (classinfo *) superclass              */
3354
3355                         /*  superclass is an interface:
3356                          *
3357                          *  return (sub != NULL) &&
3358                          *         (sub->vftbl->interfacetablelength > super->index) &&
3359                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3360                          *
3361                          *  superclass is a class:
3362                          *
3363                          *  return ((sub != NULL) && (0
3364                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3365                          *          super->vftbl->diffvall));
3366                          */
3367
3368                         {
3369                         classinfo *super;
3370                         vftbl_t   *supervftbl;
3371                         s4         superindex;
3372
3373                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3374                                 super = NULL;
3375                                 superindex = 0;
3376                                 supervftbl = NULL;
3377
3378                         } else {
3379                                 super = iptr->sx.s23.s3.c.cls;
3380                                 superindex = super->index;
3381                                 supervftbl = super->vftbl;
3382                         }
3383
3384 #if defined(ENABLE_THREADS)
3385             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3386 #endif
3387
3388                         s1 = emit_load_s1(jd, iptr, REG_ITMP1);
3389                         d = codegen_reg_of_dst(jd, iptr, REG_ITMP2);
3390                         if (s1 == d) {
3391                                 M_INTMOVE(s1, REG_ITMP1);
3392                                 s1 = REG_ITMP1;
3393                         }
3394
3395                         /* calculate interface instanceof code size */
3396
3397                         s2 = 3; /* mov_membase_reg */
3398                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3399                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3400                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3401                                 3 /* test */ + 4 /* setcc */;
3402
3403                         if (!super)
3404                                 s2 += (opt_showdisassemble ? 5 : 0);
3405
3406                         /* calculate class instanceof code size */
3407
3408                         s3 = 3; /* mov_membase_reg */
3409                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3410                         s3 += 10; /* mov_imm_reg */
3411                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3412                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3413                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3414                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3415                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3416                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3417                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3418
3419                         if (!super)
3420                                 s3 += (opt_showdisassemble ? 5 : 0);
3421
3422                         emit_alu_reg_reg(cd, ALU_XOR, d, d);
3423
3424                         /* if class is not resolved, check which code to call */
3425
3426                         if (!super) {
3427                                 emit_test_reg_reg(cd, s1, s1);
3428                                 emit_jcc(cd, CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3429                                                                                          7 + 6 + s2 + 5 + s3));
3430
3431                                 codegen_addpatchref(cd, PATCHER_checkcast_instanceof_flags,
3432                                                                         iptr->sx.s23.s3.c.ref, 0);
3433
3434                                 if (opt_showdisassemble) {
3435                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3436                                 }
3437
3438                                 emit_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3439                                 emit_alul_imm_reg(cd, ALU_AND, ACC_INTERFACE, REG_ITMP3);
3440                                 emit_jcc(cd, CC_Z, s2 + 5);
3441                         }
3442
3443                         /* interface instanceof code */
3444
3445                         if (!super || (super->flags & ACC_INTERFACE)) {
3446                                 if (super) {
3447                                         emit_test_reg_reg(cd, s1, s1);
3448                                         emit_jcc(cd, CC_Z, s2);
3449                                 }
3450
3451                                 emit_mov_membase_reg(cd, s1,
3452                                                                            OFFSET(java_objectheader, vftbl),
3453                                                                            REG_ITMP1);
3454                                 if (!super) {
3455                                         codegen_addpatchref(cd,
3456                                                                                 PATCHER_checkcast_instanceof_interface,
3457                                                                                 iptr->sx.s23.s3.c.ref, 0);
3458
3459                                         if (opt_showdisassemble) {
3460                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3461                                         }
3462                                 }
3463
3464                                 emit_movl_membase32_reg(cd, REG_ITMP1,
3465                                                                                   OFFSET(vftbl_t, interfacetablelength),
3466                                                                                   REG_ITMP3);
3467                                 emit_alu_imm32_reg(cd, ALU_SUB, superindex, REG_ITMP3);
3468                                 emit_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3469
3470                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3471
3472                                 emit_jcc(cd, CC_LE, a);
3473                                 emit_mov_membase32_reg(cd, REG_ITMP1,
3474                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3475                                                                                  superindex * sizeof(methodptr*),
3476                                                                                  REG_ITMP1);
3477                                 emit_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3478                                 emit_setcc_reg(cd, CC_NE, d);
3479
3480                                 if (!super)
3481                                         emit_jmp_imm(cd, s3);
3482                         }
3483
3484                         /* class instanceof code */
3485
3486                         if (!super || !(super->flags & ACC_INTERFACE)) {
3487                                 if (super) {
3488                                         emit_test_reg_reg(cd, s1, s1);
3489                                         emit_jcc(cd, CC_E, s3);
3490                                 }
3491
3492                                 emit_mov_membase_reg(cd, s1,
3493                                                                            OFFSET(java_objectheader, vftbl),
3494                                                                            REG_ITMP1);
3495
3496                                 if (!super) {
3497                                         codegen_addpatchref(cd, PATCHER_instanceof_class,
3498                                                                                 iptr->sx.s23.s3.c.ref, 0);
3499
3500                                         if (opt_showdisassemble) {
3501                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3502                                         }
3503                                 }
3504
3505                                 emit_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3506 #if defined(ENABLE_THREADS)
3507                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3508 #endif
3509                                 emit_movl_membase_reg(cd, REG_ITMP1,
3510                                                                                 OFFSET(vftbl_t, baseval),
3511                                                                                 REG_ITMP1);
3512                                 emit_movl_membase_reg(cd, REG_ITMP2,
3513                                                                                 OFFSET(vftbl_t, diffval),
3514                                                                                 REG_ITMP3);
3515                                 emit_movl_membase_reg(cd, REG_ITMP2,
3516                                                                                 OFFSET(vftbl_t, baseval),
3517                                                                                 REG_ITMP2);
3518 #if defined(ENABLE_THREADS)
3519                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3520 #endif
3521                                 emit_alu_reg_reg(cd, ALU_SUB, REG_ITMP2, REG_ITMP1);
3522                                 emit_alu_reg_reg(cd, ALU_XOR, d, d); /* may be REG_ITMP2 */
3523                                 emit_alu_reg_reg(cd, ALU_CMP, REG_ITMP3, REG_ITMP1);
3524                                 emit_setcc_reg(cd, CC_BE, d);
3525                         }
3526                         emit_store_dst(jd, iptr, d);
3527                         }
3528                         break;
3529
3530                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3531
3532                         /* check for negative sizes and copy sizes to stack if necessary  */
3533
3534                         MCODECHECK((10 * 4 * iptr->s1.argcount) + 5 + 10 * 8);
3535
3536                         for (s1 = iptr->s1.argcount; --s1 >= 0; ) {
3537
3538                                 src = iptr->sx.s23.s2.args[s1];
3539
3540                                 /* copy SAVEDVAR sizes to stack */
3541
3542                                 if (src->varkind != ARGVAR) {
3543                                         s2 = emit_load(jd, iptr, src, REG_ITMP1);
3544                                         M_LST(s2, REG_SP, s1 * 8);
3545                                 }
3546                         }
3547
3548                         /* is a patcher function set? */
3549
3550                         if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
3551                                 codegen_addpatchref(cd, PATCHER_builtin_multianewarray,
3552                                                                         iptr->sx.s23.s3.c.ref, 0);
3553
3554                                 if (opt_showdisassemble) {
3555                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3556                                 }
3557                         }
3558
3559                         /* a0 = dimension count */
3560
3561                         M_MOV_IMM(iptr->s1.argcount, rd->argintregs[0]);
3562
3563                         /* a1 = classinfo */
3564
3565                         M_MOV_IMM(iptr->sx.s23.s3.c.cls, rd->argintregs[1]);
3566
3567                         /* a2 = pointer to dimensions = stack pointer */
3568
3569                         M_MOV(REG_SP, rd->argintregs[2]);
3570
3571                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3572                         M_CALL(REG_ITMP1);
3573
3574                         /* check for exception before result assignment */
3575
3576                         M_TEST(REG_RESULT);
3577                         M_BEQ(0);
3578                         codegen_add_fillinstacktrace_ref(cd);
3579
3580                         s1 = codegen_reg_of_dst(jd, iptr, REG_RESULT);
3581                         M_INTMOVE(REG_RESULT, s1);
3582                         emit_store_dst(jd, iptr, s1);
3583                         break;
3584
3585                 default:
3586                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3587                         return false;
3588         } /* switch */
3589
3590         } /* for instruction */
3591
3592         /* copy values to interface registers */
3593
3594         len = bptr->outdepth;
3595         MCODECHECK(512);
3596 #if defined(ENABLE_LSRA)
3597         if (!opt_lsra)
3598 #endif
3599         while (len) {
3600                 len--;
3601                 src = bptr->outvars[len];
3602                 if ((src->varkind != STACKVAR)) {
3603                         s2 = src->type;
3604                         if (IS_FLT_DBL_TYPE(s2)) {
3605                                 s1 = emit_load(jd, iptr, src, REG_FTMP1);
3606                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3607                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3608
3609                                 } else {
3610                                         emit_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3611                                 }
3612
3613                         } else {
3614                                 s1 = emit_load(jd, iptr, src, REG_ITMP1);
3615                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3616                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3617
3618                                 } else {
3619                                         emit_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3620                                 }
3621                         }
3622                 }
3623         }
3624
3625         /* At the end of a basic block we may have to append some nops,
3626            because the patcher stub calling code might be longer than the
3627            actual instruction. So codepatching does not change the
3628            following block unintentionally. */
3629
3630         if (cd->mcodeptr < cd->lastmcodeptr) {
3631                 while (cd->mcodeptr < cd->lastmcodeptr) {
3632                         M_NOP;
3633                 }
3634         }
3635
3636         } /* if (bptr -> flags >= BBREACHED) */
3637         } /* for basic block */
3638
3639         dseg_createlinenumbertable(cd);
3640
3641         /* generate stubs */
3642
3643         emit_exception_stubs(jd);
3644         emit_patcher_stubs(jd);
3645 #if 0
3646         emit_replacement_stubs(jd);
3647 #endif
3648
3649         codegen_finish(jd);
3650
3651         /* everything's ok */
3652
3653         return true;
3654 }
3655
3656
3657 /* createcompilerstub **********************************************************
3658
3659    Creates a stub routine which calls the compiler.
3660
3661 *******************************************************************************/
3662
3663 #define COMPILERSTUB_DATASIZE    3 * SIZEOF_VOID_P
3664 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
3665
3666 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
3667
3668
3669 u1 *createcompilerstub(methodinfo *m)
3670 {
3671         u1          *s;                     /* memory to hold the stub            */
3672         ptrint      *d;
3673         codeinfo    *code;
3674         codegendata *cd;
3675         s4           dumpsize;
3676
3677         s = CNEW(u1, COMPILERSTUB_SIZE);
3678
3679         /* set data pointer and code pointer */
3680
3681         d = (ptrint *) s;
3682         s = s + COMPILERSTUB_DATASIZE;
3683
3684         /* mark start of dump memory area */
3685
3686         dumpsize = dump_size();
3687
3688         cd = DNEW(codegendata);
3689         cd->mcodeptr = s;
3690
3691         /* Store the codeinfo pointer in the same place as in the
3692            methodheader for compiled methods. */
3693
3694         code = code_codeinfo_new(m);
3695
3696         d[0] = (ptrint) asm_call_jit_compiler;
3697         d[1] = (ptrint) m;
3698         d[2] = (ptrint) code;
3699
3700         /* code for the stub */
3701
3702         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 2 * SIZEOF_VOID_P));       /* methodinfo  */
3703         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 3 * SIZEOF_VOID_P));  /* compiler pointer */
3704         M_JMP(REG_ITMP3);
3705
3706 #if defined(ENABLE_STATISTICS)
3707         if (opt_stat)
3708                 count_cstub_len += COMPILERSTUB_SIZE;
3709 #endif
3710
3711         /* release dump area */
3712
3713         dump_release(dumpsize);
3714
3715         return s;
3716 }
3717
3718
3719 /* createnativestub ************************************************************
3720
3721    Creates a stub routine which calls a native method.
3722
3723 *******************************************************************************/
3724
3725 u1 *createnativestub(functionptr f, jitdata *jd, methoddesc *nmd)
3726 {
3727         methodinfo   *m;
3728         codeinfo     *code;
3729         codegendata  *cd;
3730         registerdata *rd;
3731         methoddesc   *md;
3732         s4            nativeparams;
3733         s4            i, j;                 /* count variables                    */
3734         s4            t;
3735         s4            s1, s2;
3736
3737         /* get required compiler data */
3738
3739         m    = jd->m;
3740         code = jd->code;
3741         cd   = jd->cd;
3742         rd   = jd->rd;
3743
3744         /* initialize variables */
3745
3746         md = m->parseddesc;
3747         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
3748
3749         /* calculate stack frame size */
3750
3751         cd->stackframesize =
3752                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
3753                 sizeof(localref_table) / SIZEOF_VOID_P +
3754                 INT_ARG_CNT + FLT_ARG_CNT +
3755                 1 +                       /* functionptr, TODO: store in data segment */
3756                 nmd->memuse;
3757
3758         cd->stackframesize |= 0x1;                  /* keep stack 16-byte aligned */
3759
3760         /* create method header */
3761
3762         (void) dseg_addaddress(cd, code);                      /* CodeinfoPointer */
3763         (void) dseg_adds4(cd, cd->stackframesize * 8);         /* FrameSize       */
3764         (void) dseg_adds4(cd, 0);                              /* IsSync          */
3765         (void) dseg_adds4(cd, 0);                              /* IsLeaf          */
3766         (void) dseg_adds4(cd, 0);                              /* IntSave         */
3767         (void) dseg_adds4(cd, 0);                              /* FltSave         */
3768         (void) dseg_addlinenumbertablesize(cd);
3769         (void) dseg_adds4(cd, 0);                              /* ExTableSize     */
3770
3771         /* generate native method profiling code */
3772
3773         if (JITDATA_HAS_FLAG_INSTRUMENT(jd)) {
3774                 /* count frequency */
3775
3776                 M_MOV_IMM(code, REG_ITMP3);
3777                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
3778         }
3779
3780         /* generate stub code */
3781
3782         M_ASUB_IMM(cd->stackframesize * 8, REG_SP);
3783
3784 #if !defined(NDEBUG)
3785         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3786                 emit_verbosecall_enter(jd);
3787 #endif
3788
3789         /* get function address (this must happen before the stackframeinfo) */
3790
3791 #if !defined(WITH_STATIC_CLASSPATH)
3792         if (f == NULL) {
3793                 codegen_addpatchref(cd, PATCHER_resolve_native, m, 0);
3794
3795                 if (opt_showdisassemble) {
3796                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3797                 }
3798         }
3799 #endif
3800
3801         M_MOV_IMM(f, REG_ITMP3);
3802
3803
3804         /* save integer and float argument registers */
3805
3806         for (i = 0, j = 0; i < md->paramcount; i++) {
3807                 if (!md->params[i].inmemory) {
3808                         s1 = md->params[i].regoff;
3809
3810                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3811                                 M_LST(rd->argintregs[s1], REG_SP, j * 8);
3812                         else
3813                                 M_DST(rd->argfltregs[s1], REG_SP, j * 8);
3814
3815                         j++;
3816                 }
3817         }
3818
3819         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3820
3821         /* create dynamic stack info */
3822
3823         M_ALEA(REG_SP, cd->stackframesize * 8, rd->argintregs[0]);
3824         emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase), rd->argintregs[1]);
3825         M_ALEA(REG_SP, cd->stackframesize * 8 + SIZEOF_VOID_P, rd->argintregs[2]);
3826         M_ALD(rd->argintregs[3], REG_SP, cd->stackframesize * 8);
3827         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
3828         M_CALL(REG_ITMP1);
3829
3830         /* restore integer and float argument registers */
3831
3832         for (i = 0, j = 0; i < md->paramcount; i++) {
3833                 if (!md->params[i].inmemory) {
3834                         s1 = md->params[i].regoff;
3835
3836                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
3837                                 M_LLD(rd->argintregs[s1], REG_SP, j * 8);
3838                         else
3839                                 M_DLD(rd->argfltregs[s1], REG_SP, j * 8);
3840
3841                         j++;
3842                 }
3843         }
3844
3845         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
3846
3847
3848         /* copy or spill arguments to new locations */
3849
3850         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
3851                 t = md->paramtypes[i].type;
3852
3853                 if (IS_INT_LNG_TYPE(t)) {
3854                         if (!md->params[i].inmemory) {
3855                                 s1 = rd->argintregs[md->params[i].regoff];
3856
3857                                 if (!nmd->params[j].inmemory) {
3858                                         s2 = rd->argintregs[nmd->params[j].regoff];
3859                                         M_INTMOVE(s1, s2);
3860
3861                                 } else {
3862                                         s2 = nmd->params[j].regoff;
3863                                         M_LST(s1, REG_SP, s2 * 8);
3864                                 }
3865
3866                         } else {
3867                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3868                                 s2 = nmd->params[j].regoff;
3869                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
3870                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
3871                         }
3872
3873                 } else {
3874                         /* We only copy spilled float arguments, as the float argument    */
3875                         /* registers keep unchanged.                                      */
3876
3877                         if (md->params[i].inmemory) {
3878                                 s1 = md->params[i].regoff + cd->stackframesize + 1;   /* + 1 (RA) */
3879                                 s2 = nmd->params[j].regoff;
3880
3881                                 if (IS_2_WORD_TYPE(t)) {
3882                                         M_DLD(REG_FTMP1, REG_SP, s1 * 8);
3883                                         M_DST(REG_FTMP1, REG_SP, s2 * 8);
3884                                 } else {
3885                                         M_FLD(REG_FTMP1, REG_SP, s1 * 8);
3886                                         M_FST(REG_FTMP1, REG_SP, s2 * 8);
3887                                 }
3888                         }
3889                 }
3890         }
3891
3892         /* put class into second argument register */
3893
3894         if (m->flags & ACC_STATIC)
3895                 M_MOV_IMM(m->class, rd->argintregs[1]);
3896
3897         /* put env into first argument register */
3898
3899         M_MOV_IMM(_Jv_env, rd->argintregs[0]);
3900
3901         /* do the native function call */
3902
3903         M_CALL(REG_ITMP3);
3904
3905         /* save return value */
3906
3907         if (md->returntype.type != TYPE_VOID) {
3908                 if (IS_INT_LNG_TYPE(md->returntype.type))
3909                         M_LST(REG_RESULT, REG_SP, 0 * 8);
3910                 else
3911                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
3912         }
3913
3914 #if !defined(NDEBUG)
3915         if (JITDATA_HAS_FLAG_VERBOSECALL(jd))
3916                 emit_verbosecall_exit(jd);
3917 #endif
3918
3919         /* remove native stackframe info */
3920
3921         M_ALEA(REG_SP, cd->stackframesize * 8, rd->argintregs[0]);
3922         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
3923         M_CALL(REG_ITMP1);
3924         M_MOV(REG_RESULT, REG_ITMP3);
3925
3926         /* restore return value */
3927
3928         if (md->returntype.type != TYPE_VOID) {
3929                 if (IS_INT_LNG_TYPE(md->returntype.type))
3930                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
3931                 else
3932                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
3933         }
3934
3935         /* remove stackframe */
3936
3937         M_AADD_IMM(cd->stackframesize * 8, REG_SP);
3938
3939         /* test for exception */
3940
3941         M_TEST(REG_ITMP3);
3942         M_BNE(1);
3943         M_RET;
3944
3945         /* handle exception */
3946
3947         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
3948         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
3949         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
3950
3951         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
3952         M_JMP(REG_ITMP3);
3953
3954         /* generate patcher stubs */
3955
3956         emit_patcher_stubs(jd);
3957
3958         codegen_finish(jd);
3959
3960         return code->entrypoint;
3961 }
3962
3963
3964 /*
3965  * These are local overrides for various environment variables in Emacs.
3966  * Please do not remove this and leave it at the end of the file, where
3967  * Emacs will automagically detect them.
3968  * ---------------------------------------------------------------------
3969  * Local variables:
3970  * mode: c
3971  * indent-tabs-mode: t
3972  * c-basic-offset: 4
3973  * tab-width: 4
3974  * End:
3975  * vim:noexpandtab:sw=4:ts=4:
3976  */